kanshan/TensorRT-LLMs
1
0
Fork 0
mirror of https://github.com/NVIDIA/TensorRT-LLM.git synced 2026-01-14 06:27:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat: Make benchmark_serving part of the library (#5428)

Browse Source 
Signed-off-by: Kaiyu Xie <26294424+kaiyux@users.noreply.github.com>
This commit is contained in:
Kaiyu Xie 2025-06-25 23:13:56 +08:00 committed by GitHub
parent 314f15f0a7
commit c5ae3272b9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 12 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
tensorrt_llm/serve/scripts/__init__.py Normal file
View File

3
tensorrt_llm/serve/scripts/benchmark_dataset.py
View File

@ -26,10 +26,11 @@ from typing import Any, Callable, Optional, Union
import numpy as np
import pandas as pd
from benchmark_utils import download_and_cache_file
from datasets import load_dataset
from transformers import PreTrainedTokenizerBase
from .benchmark_utils import download_and_cache_file
logger = logging.getLogger(__name__)
# -----------------------------------------------------------------------------

19
tensorrt_llm/serve/scripts/benchmark_serving.py
View File

@ -31,18 +31,19 @@ from datetime import datetime
from typing import Any, Optional
import numpy as np
from backend_request_func import (ASYNC_REQUEST_FUNCS,
OPENAI_COMPATIBLE_BACKENDS, RequestFuncInput,
RequestFuncOutput, get_tokenizer)
from benchmark_dataset import (AIMODataset, BurstGPTDataset,
ConversationDataset, HuggingFaceDataset,
InstructCoderDataset, RandomDataset,
SampleRequest, ShareGPTDataset, SonnetDataset,
VisionArenaDataset)
from benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
from tqdm.asyncio import tqdm
from transformers import PreTrainedTokenizerBase
from .backend_request_func import (ASYNC_REQUEST_FUNCS,
OPENAI_COMPATIBLE_BACKENDS, RequestFuncInput,
RequestFuncOutput, get_tokenizer)
from .benchmark_dataset import (AIMODataset, BurstGPTDataset,
ConversationDataset, HuggingFaceDataset,
InstructCoderDataset, RandomDataset,
SampleRequest, ShareGPTDataset, SonnetDataset,
VisionArenaDataset)
from .benchmark_utils import convert_to_pytorch_benchmark_format, write_to_json
MILLISECONDS_TO_SECONDS_CONVERSION = 1000