mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-02-21 10:15:46 +08:00
294 lines
7.3 KiB
YAML
294 lines
7.3 KiB
YAML
server_configs:
|
|
- name: "r1_fp8_dep8_mtp1_1k1k"
|
|
model_name: "deepseek_r1_0528_fp8"
|
|
gpus: 8
|
|
tensor_parallel_size: 8
|
|
moe_expert_parallel_size: 8
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 512
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: true
|
|
attention_dp_config:
|
|
batching_wait_iters: 0
|
|
enable_balance: true
|
|
timeout_iters: 60
|
|
moe_config:
|
|
backend: 'DEEPGEMM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 512
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 1
|
|
client_configs:
|
|
- name: "con4096_iter10_1k1k"
|
|
concurrency: 4096
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "r1_fp8_tep8_mtp3_1k1k"
|
|
model_name: "deepseek_r1_0528_fp8"
|
|
gpus: 8
|
|
tensor_parallel_size: 8
|
|
moe_expert_parallel_size: 8
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 64
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: false
|
|
moe_config:
|
|
backend: 'DEEPGEMM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 64
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 3
|
|
client_configs:
|
|
- name: "con64_iter10_1k1k"
|
|
concurrency: 64
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "r1_fp8_tp8_mtp3_1k1k"
|
|
model_name: "deepseek_r1_0528_fp8"
|
|
gpus: 8
|
|
tensor_parallel_size: 8
|
|
moe_expert_parallel_size: 1
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 8
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: false
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 8
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 3
|
|
client_configs:
|
|
- name: "con8_iter10_1k1k"
|
|
concurrency: 8
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "r1_fp4_v2_dep4_mtp1_1k1k"
|
|
model_name: "deepseek_r1_0528_fp4_v2"
|
|
gpus: 4
|
|
tensor_parallel_size: 4
|
|
moe_expert_parallel_size: 4
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 512
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: true
|
|
attention_dp_config:
|
|
batching_wait_iters: 0
|
|
enable_balance: true
|
|
timeout_iters: 60
|
|
moe_config:
|
|
backend: 'CUTLASS'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 512
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 1
|
|
client_configs:
|
|
- name: "con2048_iter10_1k1k"
|
|
concurrency: 2048
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "r1_fp4_v2_tep4_mtp3_1k1k"
|
|
model_name: "deepseek_r1_0528_fp4_v2"
|
|
gpus: 4
|
|
tensor_parallel_size: 4
|
|
moe_expert_parallel_size: 4
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 32
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: false
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 32
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 3
|
|
client_configs:
|
|
- name: "con32_iter10_1k1k"
|
|
concurrency: 32
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "r1_fp4_v2_tp4_mtp3_1k1k"
|
|
model_name: "deepseek_r1_0528_fp4_v2"
|
|
gpus: 4
|
|
tensor_parallel_size: 4
|
|
moe_expert_parallel_size: 1
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 4
|
|
max_num_tokens: 8192
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: false
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 4
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'MTP'
|
|
num_nextn_predict_layers: 3
|
|
client_configs:
|
|
- name: "con4_iter10_1k1k"
|
|
concurrency: 4
|
|
iterations: 10
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "gpt_oss_fp4_dep2_1k1k"
|
|
model_name: "gpt_oss_120b_fp4"
|
|
gpus: 2
|
|
tensor_parallel_size: 2
|
|
moe_expert_parallel_size: 2
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 1024
|
|
max_num_tokens: 20000
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: true
|
|
attention_dp_config:
|
|
enable_balance: true
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 1024
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
num_postprocess_workers: 4
|
|
stream_interval: 20
|
|
client_configs:
|
|
- name: "con2048_iter5_1k1k"
|
|
concurrency: 2048
|
|
iterations: 5
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "gpt_oss_fp4_dep4_1k1k"
|
|
model_name: "gpt_oss_120b_fp4"
|
|
gpus: 4
|
|
tensor_parallel_size: 4
|
|
moe_expert_parallel_size: 4
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 512
|
|
max_num_tokens: 20000
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: true
|
|
attention_dp_config:
|
|
enable_balance: true
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 512
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
num_postprocess_workers: 4
|
|
stream_interval: 20
|
|
client_configs:
|
|
- name: "con2048_iter5_1k1k"
|
|
concurrency: 2048
|
|
iterations: 5
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|
|
|
|
- name: "gpt_oss_fp4_tp4_eagle3_1k1k"
|
|
model_name: "gpt_oss_120b_fp4"
|
|
gpus: 4
|
|
tensor_parallel_size: 4
|
|
moe_expert_parallel_size: 1
|
|
pipeline_parallel_size: 1
|
|
max_batch_size: 1
|
|
max_num_tokens: 20000
|
|
attn_backend: "TRTLLM"
|
|
enable_attention_dp: false
|
|
moe_config:
|
|
backend: 'TRTLLM'
|
|
cuda_graph_config:
|
|
enable_padding: true
|
|
max_batch_size: 1
|
|
kv_cache_config:
|
|
dtype: 'fp8'
|
|
enable_block_reuse: false
|
|
free_gpu_memory_fraction: 0.8
|
|
speculative_config:
|
|
decoding_type: 'Eagle'
|
|
eagle3_layers_to_capture: [-1]
|
|
max_draft_len: 3
|
|
speculative_model_dir: "gpt_oss/gpt-oss-120b-Eagle3"
|
|
stream_interval: 20
|
|
num_postprocess_workers: 4
|
|
client_configs:
|
|
- name: "con1_iter32_1k1k"
|
|
concurrency: 1
|
|
iterations: 32
|
|
isl: 1024
|
|
osl: 1024
|
|
random_range_ratio: 0.2
|
|
backend: "openai"
|