mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-14 06:27:45 +08:00
5955b8afba
* Update TensorRT-LLM --------- Co-authored-by: Shixiaowei02 <39303645+Shixiaowei02@users.noreply.github.com>
17 lines
360 B
Bash
17 lines
360 B
Bash
#!/bin/bash
|
|
set -ex
|
|
|
|
PROMPT="Tell a story"
|
|
LLAMA_MODEL_DIR=$1
|
|
|
|
|
|
python3 llm_examples.py --task run_llm_with_quantization \
|
|
--prompt="$PROMPT" \
|
|
--hf_model_dir=$LLAMA_MODEL_DIR \
|
|
--quant_type="int4_awq"
|
|
|
|
python3 llm_examples.py --task run_llm_with_quantization \
|
|
--prompt="$PROMPT" \
|
|
--hf_model_dir=$LLAMA_MODEL_DIR \
|
|
--quant_type="fp8"
|