mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-14 06:27:45 +08:00
35 lines
873 B
Bash
35 lines
873 B
Bash
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# Parse arguments
|
|
full_logdir=${1}
|
|
accuracy_model=${2}
|
|
accuracy_tasks=${3}
|
|
model_path=${4}
|
|
model_args_extra=${5}
|
|
output_dir=${6}
|
|
hostname=${7}
|
|
port=${8}
|
|
|
|
echo "Starting accuracy evaluation..."
|
|
echo "Log directory: ${full_logdir}"
|
|
|
|
echo "Hostname: ${hostname}, Port: ${port}"
|
|
base_url="http://${hostname}:${port}/v1/completions"
|
|
echo "Using base_url: ${base_url}"
|
|
|
|
# Install lm_eval and run evaluation
|
|
echo "Installing lm_eval[api] and running evaluation..."
|
|
pip install lm_eval[api]==0.4.8
|
|
|
|
echo "Running lm_eval with tasks: ${accuracy_tasks}..."
|
|
|
|
mkdir -p ${output_dir}
|
|
lm_eval --model ${accuracy_model} \
|
|
--tasks ${accuracy_tasks} \
|
|
--model_args model=${model_path},base_url=${base_url},${model_args_extra} \
|
|
--output_path ${output_dir} --log_samples \
|
|
--trust_remote_code
|
|
|
|
echo "Accuracy evaluation completed successfully"
|