mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-02-04 02:02:01 +08:00
b2f69db507
* add eval_llmapi Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> tmp commit port to CLI tool Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> move Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> setup llmapi Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> fix spec_dec_algo Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> _update_from_hf_quant_config Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> fix Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> migrate test_pytorch.py Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> fix fp8 block scales Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> fix fp8 rowwise Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> adj alpha Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> move test_pytorch.py cases Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> move Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> rename test_accuracy.py to test_cli.py Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> clean Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * fix cnn_dailymail Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * renaming to cli flow Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * rename MMLU Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * rename Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * add error Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> * fix Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com> --------- Signed-off-by: Enwei Zhu <21126786+syuoni@users.noreply.github.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
import argparse
|
|
import re
|
|
|
|
import pandas as pd
|
|
|
|
metric_regex = {
|
|
"rouge1": r"(?<=rouge1: )\d+\.\d+",
|
|
"perplexity": r"(?<=Per-token perplexity: )\d+\.\d+",
|
|
"mmlu": r"(?<=MMLU weighted average accuracy: )\d+\.\d+",
|
|
"passkey": r"(?<=passkey accuracy: )\d+\.\d+"
|
|
}
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--log_file", type=str, required=True)
|
|
parser.add_argument("--output_file", type=str, default=None)
|
|
args = parser.parse_args()
|
|
|
|
with open(args.log_file) as f:
|
|
log = f.read()
|
|
|
|
test_cases = re.search(r"(?<=items in this shard: ).+", log).group()
|
|
test_cases = test_cases.split(", ")
|
|
data = [{} for _ in test_cases]
|
|
|
|
log = log.split("\n")
|
|
i = -1
|
|
for line in log:
|
|
if i + 1 < len(test_cases) and line.startswith(test_cases[i + 1]):
|
|
# Advance to next test case
|
|
i += 1
|
|
continue
|
|
if i < 0:
|
|
continue
|
|
|
|
entry = data[i]
|
|
for metric, regex in metric_regex.items():
|
|
if metric in entry:
|
|
continue
|
|
matched = re.search(regex, line)
|
|
if matched:
|
|
entry[metric] = float(matched.group())
|
|
|
|
df = pd.DataFrame(data, index=test_cases)
|
|
print(df)
|
|
|
|
if args.output_file:
|
|
df.to_csv(args.output_file)
|