mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-02-11 21:43:24 +08:00
Update latest GitHub pages to v1.2.0rc1
This commit is contained in:
Kaiyu Xie
parent
de03512fa6
commit
72a4b6677e
@ -1,4 +1,4 @@
|
||||
# Sphinx build info version 1
|
||||
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||
config: 05441684cb2c0903bdac9ebb5abe267d
|
||||
config: eb18464cd19c763f9cb542fdd6f60977
|
||||
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
||||
|
||||
@ -59,7 +59,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -69,7 +69,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -335,6 +335,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -365,6 +366,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -407,6 +409,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -423,6 +426,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -5431,7 +5435,7 @@
|
||||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||||
<dl class="cpp var">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4cb2fb0a75c587a97ceabfb7556bb4f1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxSeqIdleMicroseconds</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">180000000</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE" title="Link to this definition">#</a><br /></dt>
|
||||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4cb2fb0a75c587a97ceabfb7556bb4f1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxSeqIdleMicroseconds</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">duration_cast</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">microseconds</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">(</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">minutes</span></span><span class="p"><span class="pre">(</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">.</span></span><span class="n"><span class="pre">count</span></span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp var">
|
||||
@ -13755,9 +13759,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -59,7 +59,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -69,7 +69,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -335,6 +335,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -365,6 +366,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -407,6 +409,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -423,6 +426,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -3065,7 +3069,7 @@ one more than decoding draft tokens for prediction from primary head </p>
|
||||
|
||||
<dl class="cpp function">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv">
|
||||
<span id="_CPPv3NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="tensorrt_llm::runtime::LookaheadModule::getExecutionConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1ad81b2560fd286eb36d5083279cd13f13"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getExecutionConfig</span></span></span><span class="sig-paren">(</span>
|
||||
<span id="_CPPv3NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="tensorrt_llm::runtime::LookaheadModule::getExecutionConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1a95cca340c59dc6e72d968c6ccfeada6e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getExecutionConfig</span></span></span><span class="sig-paren">(</span>
|
||||
|
||||
<dl>
|
||||
</dl>
|
||||
@ -14756,9 +14760,9 @@ one more than decoding draft tokens for prediction from primary head </p>
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -127,6 +127,8 @@ class Attention(nn.Module):
|
||||
q_scaling: float = 1.0,
|
||||
attention_chunk_size: Optional[int] = None,
|
||||
disable_deep_gemm: bool = False,
|
||||
attn_output_gate: Optional[bool] = None,
|
||||
use_custom_cublas_mm: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize the Attention module.
|
||||
@ -146,6 +148,7 @@ class Attention(nn.Module):
|
||||
q_scaling (float): The scaling factor for the qk_scale. The definition is $O = softmax(QK^T * qk_scale) * V, qk_scale = 1 / (sqrt(head_dim) * q_scaling)$. The default value is 1.0.
|
||||
attention_chunk_size (Optional[int]): See [Chunked Attention] below.
|
||||
disable_deep_gemm (bool): Whether to disable the use of DeepGEMM in Linear layers (currently only matters on SM100 + FP8).
|
||||
attn_output_gate (Optional[bool]): Determines whether to use an output gate in the attention Op. If False, the decision is automatically handled by the attention backend based on its capabilities.
|
||||
"""
|
||||
super().__init__()
|
||||
self.layer_idx = layer_idx
|
||||
@ -172,6 +175,10 @@ class Attention(nn.Module):
|
||||
self.pos_embd_params = pos_embd_params
|
||||
self.dense_bias = dense_bias
|
||||
self.q_scaling = q_scaling
|
||||
self.attn_output_gate = attn_output_gate
|
||||
|
||||
if self.attn_output_gate:
|
||||
logger.info_once("using attn output gate!", key="attn_output_gate")
|
||||
|
||||
# [Chunked Attention]
|
||||
# Chunked attention is applied to context requests only. Chunked attention will be
|
||||
@ -217,7 +224,8 @@ class Attention(nn.Module):
|
||||
|
||||
self.qkv_proj = Linear(
|
||||
self.hidden_size,
|
||||
tp_size * self.q_size + 2 * tp_size * self.kv_size,
|
||||
tp_size * self.q_size * (2 if self.attn_output_gate else 1) +
|
||||
2 * tp_size * self.kv_size,
|
||||
bias=bias,
|
||||
dtype=dtype,
|
||||
mapping=mapping,
|
||||
@ -229,7 +237,7 @@ class Attention(nn.Module):
|
||||
allreduce_strategy=config.allreduce_strategy,
|
||||
force_dynamic_quantization=config.force_dynamic_quantization,
|
||||
disable_deep_gemm=disable_deep_gemm,
|
||||
)
|
||||
use_custom_cublas_mm=use_custom_cublas_mm)
|
||||
|
||||
self.o_lora = LoraLayer([LoraModuleType.ATTENTION_DENSE],
|
||||
[self.hidden_size])
|
||||
@ -247,11 +255,13 @@ class Attention(nn.Module):
|
||||
allreduce_strategy=config.allreduce_strategy,
|
||||
force_dynamic_quantization=config.force_dynamic_quantization,
|
||||
disable_deep_gemm=disable_deep_gemm,
|
||||
)
|
||||
use_custom_cublas_mm=use_custom_cublas_mm)
|
||||
|
||||
self.quant_config = config.get_quant_config()
|
||||
self.attn_backend = config.attn_backend
|
||||
attn_cls = get_attention_backend(self.attn_backend)
|
||||
attn_cls = get_attention_backend(
|
||||
self.attn_backend,
|
||||
sparse_attn_config=config.sparse_attention_config)
|
||||
|
||||
# These two modules are mutually exclusive - either splitted_qkv_lora or fused_qkv_lora will be used,
|
||||
# but never both at the same time. splitted_qkv_lora handles Q,K,V separately while fused_qkv_lora
|
||||
@ -269,6 +279,9 @@ class Attention(nn.Module):
|
||||
# Whether to fuse RoPE into the attention OP.
|
||||
# If true, RoPE will be applied in self.attn.forward.
|
||||
# If false, RoPE will be applied in self.apply_rope.
|
||||
if config.sparse_attention_config is not None:
|
||||
logger.warning("disable rope_fusion for sparse attention.")
|
||||
rope_fusion = False
|
||||
self.rope_fusion = rope_fusion
|
||||
if self.rope_fusion and not attn_cls.support_fused_rope():
|
||||
logger.warning(
|
||||
@ -306,6 +319,7 @@ class Attention(nn.Module):
|
||||
skip_create_weights_in_init=config.skip_create_weights_in_init,
|
||||
q_scaling=self.q_scaling,
|
||||
attention_chunk_size=self.attention_chunk_size,
|
||||
sparse_attention_config=config.sparse_attention_config,
|
||||
)
|
||||
|
||||
self.support_fused_qkv = self.attn.support_fused_qkv()
|
||||
@ -521,24 +535,39 @@ class Attention(nn.Module):
|
||||
if qkv_lora is not None:
|
||||
qkv = qkv + qkv_lora
|
||||
|
||||
q, k, v = qkv, None, None
|
||||
if self.attn_output_gate:
|
||||
q_gate, k, v = qkv.split(
|
||||
[self.q_size * 2, self.kv_size, self.kv_size], dim=-1)
|
||||
orig_shape = q_gate.shape[:-1]
|
||||
# Single line: view -> chunk -> reshape both q and gate
|
||||
q, gate = [
|
||||
t.reshape(*orig_shape, -1) for t in torch.chunk(
|
||||
q_gate.view(*orig_shape, self.num_heads, -1), 2, dim=-1)
|
||||
]
|
||||
else:
|
||||
q, k, v = qkv, None, None
|
||||
|
||||
q, k, v = self.apply_rope(q, k, v, position_ids)
|
||||
q, k, v = self.convert_qkv(q, k, v)
|
||||
|
||||
if attention_sinks is not None:
|
||||
assert self.attn_backend == "TRTLLM", "Attention sinks are only supported for TRTLLM backend."
|
||||
|
||||
output = self.forward_impl(q,
|
||||
k,
|
||||
v,
|
||||
attn_metadata,
|
||||
attention_mask,
|
||||
attention_window_size,
|
||||
attention_mask_data,
|
||||
mrope_config=mrope_config,
|
||||
attention_sinks=attention_sinks)
|
||||
attn_output = self.forward_impl(q,
|
||||
k,
|
||||
v,
|
||||
attn_metadata,
|
||||
attention_mask,
|
||||
attention_window_size,
|
||||
attention_mask_data,
|
||||
mrope_config=mrope_config,
|
||||
attention_sinks=attention_sinks)
|
||||
|
||||
attn_output = self.o_proj(output,
|
||||
if self.attn_output_gate:
|
||||
gate = torch.sigmoid(gate)
|
||||
attn_output = attn_output * gate
|
||||
|
||||
attn_output = self.o_proj(attn_output,
|
||||
all_reduce_params=all_reduce_params,
|
||||
lora_params=lora_params,
|
||||
layer_idx=self.layer_idx)
|
||||
@ -831,6 +860,7 @@ class MLA(nn.Module):
|
||||
v_head_dim=self.v_head_dim,
|
||||
predicted_tokens_per_seq=self.predicted_tokens_per_seq,
|
||||
skip_create_weights_in_init=config.skip_create_weights_in_init,
|
||||
sparse_attention_config=config.sparse_attention_config,
|
||||
)
|
||||
|
||||
self.mqa = create_attention(
|
||||
@ -850,6 +880,7 @@ class MLA(nn.Module):
|
||||
v_head_dim=self.kv_lora_rank,
|
||||
predicted_tokens_per_seq=self.predicted_tokens_per_seq,
|
||||
skip_create_weights_in_init=config.skip_create_weights_in_init,
|
||||
sparse_attention_config=config.sparse_attention_config,
|
||||
)
|
||||
|
||||
self.aux_stream = aux_stream
|
||||
|
||||
@ -18,6 +18,8 @@ from tensorrt_llm._utils import (is_trace_enabled, nvtx_range, release_gc,
|
||||
torch_dtype_to_str, trace_func)
|
||||
from tensorrt_llm.inputs.multimodal import (MultimodalParams,
|
||||
MultimodalRuntimeData)
|
||||
from tensorrt_llm.inputs.registry import (create_input_processor,
|
||||
create_input_processor_with_hash)
|
||||
from tensorrt_llm.logger import logger
|
||||
from tensorrt_llm.lora_helper import LoraConfig
|
||||
from tensorrt_llm.lora_manager import LoraModelConfig
|
||||
@ -34,6 +36,7 @@ from ..compilation.utils import capture_piecewise_cuda_graph
|
||||
from ..distributed import MPIDist
|
||||
from ..distributed.communicator import init_pp_comm
|
||||
from ..expert_statistic import ExpertStatistic
|
||||
from ..memory_buffer_utils import with_shared_pool
|
||||
from ..metadata import KVCacheParams
|
||||
from ..models.checkpoints.base_checkpoint_loader import BaseCheckpointLoader
|
||||
from ..models.modeling_multimodal_utils import filter_mm_token_from_input_ids
|
||||
@ -61,8 +64,6 @@ from .resource_manager import (BaseResourceManager, KVCacheManager,
|
||||
from .sampler import SampleStateTensors
|
||||
from .scheduler import ScheduledRequests
|
||||
|
||||
MAX_UINT64 = (1 << 64) - 1
|
||||
|
||||
|
||||
class ModelEngine(ABC):
|
||||
|
||||
@ -139,12 +140,14 @@ class PyTorchModelEngine(ModelEngine):
|
||||
attn_runtime_features: Optional[AttentionRuntimeFeatures] = None,
|
||||
dist: Optional[MPIDist] = None,
|
||||
spec_config: Optional["DecodingBaseConfig"] = None,
|
||||
sparse_attention_config: Optional["SparseAttentionConfig"] = None,
|
||||
lora_config: Optional[LoraConfig] = None,
|
||||
is_draft_model: bool = False,
|
||||
drafting_loop_wrapper: Optional[Callable[[torch.nn.Module],
|
||||
torch.nn.Module]] = None,
|
||||
model: Optional[torch.nn.Module] = None,
|
||||
):
|
||||
self.forward_pass_callable = None
|
||||
self.ub_buffers = None
|
||||
self.batch_size = batch_size
|
||||
self.max_num_tokens = max_num_tokens
|
||||
@ -166,17 +169,21 @@ class PyTorchModelEngine(ModelEngine):
|
||||
spec_config.max_draft_len = 0
|
||||
self.spec_config = spec_config
|
||||
self.is_spec_decode = spec_config is not None
|
||||
self.sparse_attention_config = sparse_attention_config
|
||||
self.enable_spec_decode = self.is_spec_decode
|
||||
self.is_draft_model = is_draft_model
|
||||
|
||||
self.attn_runtime_features = attn_runtime_features or AttentionRuntimeFeatures(
|
||||
)
|
||||
|
||||
self.input_processor = create_input_processor(model_path, None)
|
||||
self.input_processor_with_hash = create_input_processor_with_hash(
|
||||
self.input_processor)
|
||||
if model is None:
|
||||
loader = ModelLoader(
|
||||
pytorch_backend_config=pytorch_backend_config,
|
||||
mapping=self.mapping,
|
||||
spec_config=self.spec_config,
|
||||
sparse_attention_config=self.sparse_attention_config,
|
||||
max_num_tokens=max_num_tokens,
|
||||
max_seq_len=max_seq_len,
|
||||
lora_config=lora_config,
|
||||
@ -263,7 +270,8 @@ class PyTorchModelEngine(ModelEngine):
|
||||
self.is_warmup = False
|
||||
|
||||
self.attn_backend = get_attention_backend(
|
||||
pytorch_backend_config.attn_backend)
|
||||
pytorch_backend_config.attn_backend,
|
||||
sparse_attn_config=sparse_attention_config)
|
||||
|
||||
if self.is_spec_decode:
|
||||
self.spec_metadata = None
|
||||
@ -351,6 +359,9 @@ class PyTorchModelEngine(ModelEngine):
|
||||
else:
|
||||
self.cache_indirection_attention = None
|
||||
|
||||
def register_forward_pass_callable(self, callable: Callable):
|
||||
self.forward_pass_callable = callable
|
||||
|
||||
@property
|
||||
def runtime_draft_len(self):
|
||||
return self.max_draft_len if self.enable_spec_decode else 0
|
||||
@ -446,10 +457,13 @@ class PyTorchModelEngine(ModelEngine):
|
||||
|
||||
@with_warmup_flag
|
||||
def warmup(self, resource_manager: ResourceManager) -> None:
|
||||
"""
|
||||
Orchestrates the warmup process by calling specialized warmup methods for
|
||||
torch.compile, the autotuner, and CUDA graphs.
|
||||
"""
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
|
||||
if kv_cache_manager is None:
|
||||
logger.info("Skipping warm up as no KV Cache manager allocated.")
|
||||
return
|
||||
@ -458,317 +472,394 @@ class PyTorchModelEngine(ModelEngine):
|
||||
# Reset the global cuda graph dummy request to None in warmup.
|
||||
self.cuda_graph_runner.padding_dummy_request = None
|
||||
|
||||
def get_num_extra_decoding_steps():
|
||||
if isinstance(self.model, ChainDrafter):
|
||||
return self.model.max_draft_len
|
||||
else:
|
||||
assert not self.model_is_wrapped, (
|
||||
f"Please add logic to determine num_extra_decoding_steps for drafting loop {type(self.model)}"
|
||||
)
|
||||
return 0
|
||||
# TODO: current warmup_request is not suitable for context parallelism.
|
||||
cp_type = self.mapping.cp_config.get('cp_type', None)
|
||||
if cp_type is not None:
|
||||
logger.info("[ModelEngine::warmup] Skipping warmup for cp_type: ",
|
||||
cp_type.name)
|
||||
return
|
||||
|
||||
def get_cuda_graph_warmup_request(batch_size, draft_len):
|
||||
# Divide by max_beam_width to get an approximation of the number of requests that can be run in parallel.
|
||||
available_blocks = kv_cache_manager.get_num_free_blocks(
|
||||
) // self.max_beam_width
|
||||
if available_blocks >= batch_size:
|
||||
result = ScheduledRequests()
|
||||
result.context_requests = []
|
||||
num_extra_decoding_steps = get_num_extra_decoding_steps()
|
||||
self._run_torch_compile_warmup(resource_manager)
|
||||
self._run_autotuner_warmup(resource_manager)
|
||||
self._run_cuda_graph_warmup(resource_manager)
|
||||
|
||||
# Add (batch_size - 1) dummy requests with seq_len=1.
|
||||
# Should only need one more page per request.
|
||||
requests = kv_cache_manager.add_dummy_requests(
|
||||
list(range(batch_size - 1)),
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=draft_len,
|
||||
use_mrope=self.use_mrope,
|
||||
max_beam_width=self.max_beam_width,
|
||||
num_extra_decoding_steps=num_extra_decoding_steps)
|
||||
# Divide by max_beam_width to get an approximation of the number of tokens that can be added to the final request.
|
||||
available_tokens = kv_cache_manager.get_num_available_tokens(
|
||||
draft_len)
|
||||
# Set the value back to the original value after all warmups are complete
|
||||
self.enable_spec_decode = self.is_spec_decode
|
||||
|
||||
# Add one dummy request with the maximum possible sequence length.
|
||||
# The sequence length is limited by both the max_seq_len and the number of available blocks.
|
||||
# Also, the sequence length is limited by the max_position_embeddings.
|
||||
token_num = max(1, min(available_tokens, self.max_seq_len - 1))
|
||||
model_config = self.model.model_config.pretrained_config
|
||||
max_position_embeddings = getattr(model_config,
|
||||
'max_position_embeddings',
|
||||
None)
|
||||
if max_position_embeddings is not None:
|
||||
token_num = min(token_num,
|
||||
max_position_embeddings - draft_len)
|
||||
|
||||
assert token_num > num_extra_decoding_steps, (
|
||||
"Cannot fuse drafting loop. We do not have enough KV cache space "
|
||||
"for all of the draft tokens.")
|
||||
token_num -= num_extra_decoding_steps
|
||||
|
||||
max_seq_len_request = kv_cache_manager.add_dummy_requests(
|
||||
request_ids=[batch_size - 1],
|
||||
token_nums=[token_num],
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=draft_len,
|
||||
use_mrope=self.use_mrope,
|
||||
max_beam_width=self.max_beam_width,
|
||||
num_extra_decoding_steps=num_extra_decoding_steps)[0]
|
||||
# Add the longest request before all other seq_len=1 request to simulate the padding CUDA graph case.
|
||||
# This batch contains both the longest request and the shortest requests,
|
||||
# it also contains the maximum number of requests and the maximum token number,
|
||||
# which simulates the extreme case for the padding CUDA graph.
|
||||
# Thus we can replay this CUDA graph in all other cases.
|
||||
requests.insert(0, max_seq_len_request)
|
||||
result.generation_requests = requests
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(
|
||||
request_ids=list(range(batch_size)))
|
||||
else:
|
||||
result = None
|
||||
return result
|
||||
|
||||
def get_warmup_request(num_tokens: int, num_gen_tokens: int):
|
||||
available_tokens = kv_cache_manager.get_num_available_tokens(
|
||||
self.runtime_draft_len)
|
||||
available_blocks = kv_cache_manager.get_num_free_blocks()
|
||||
if num_tokens > self.max_num_tokens or num_tokens > available_tokens:
|
||||
return None
|
||||
|
||||
num_extra_decoding_steps = get_num_extra_decoding_steps()
|
||||
if num_extra_decoding_steps > 0:
|
||||
# Disable autotuning for fused drafting loops for now.
|
||||
# There are a few bugs that can cause illegal memory accesses
|
||||
# during warmup.
|
||||
return None
|
||||
|
||||
num_ctx_tokens = num_tokens - num_gen_tokens
|
||||
num_ctx_requests = 0
|
||||
ctx_requests = []
|
||||
gen_requests = []
|
||||
|
||||
max_seq_len = self.max_seq_len - 1
|
||||
num_full_seqs = 0
|
||||
num_left_over_tokens = 0
|
||||
|
||||
if num_ctx_tokens > 0:
|
||||
# We will try to assign as less context requests as possible to
|
||||
# fill the num_ctx_tokens.
|
||||
|
||||
# Num full sequences:
|
||||
num_full_seqs = num_ctx_tokens // max_seq_len
|
||||
num_left_over_tokens = num_ctx_tokens - num_full_seqs * max_seq_len
|
||||
|
||||
num_ctx_requests = num_full_seqs + (1 if num_left_over_tokens
|
||||
> 0 else 0)
|
||||
|
||||
# We do not have enough batch to fill the request
|
||||
if num_ctx_requests + num_gen_tokens > self.batch_size:
|
||||
return None
|
||||
|
||||
blocks_to_use = num_full_seqs * math.ceil(
|
||||
max_seq_len / kv_cache_manager.tokens_per_block) + math.ceil(
|
||||
num_left_over_tokens /
|
||||
kv_cache_manager.tokens_per_block) + num_gen_tokens
|
||||
|
||||
if blocks_to_use > available_blocks:
|
||||
return None
|
||||
|
||||
if num_ctx_tokens > 0:
|
||||
ctx_token_nums = [max_seq_len] * num_full_seqs
|
||||
if num_left_over_tokens > 0:
|
||||
ctx_token_nums.append(num_left_over_tokens)
|
||||
|
||||
ctx_requests = kv_cache_manager.add_dummy_requests(
|
||||
list(range(num_ctx_requests)),
|
||||
token_nums=ctx_token_nums,
|
||||
is_gen=False,
|
||||
max_num_draft_tokens=self.runtime_draft_len,
|
||||
use_mrope=self.use_mrope)
|
||||
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(
|
||||
request_ids=list(range(num_ctx_requests)))
|
||||
|
||||
if num_gen_tokens > 0:
|
||||
gen_requests = kv_cache_manager.add_dummy_requests(
|
||||
list(
|
||||
range(num_ctx_requests,
|
||||
num_ctx_requests + num_gen_tokens)),
|
||||
token_nums=[1] * num_gen_tokens,
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=self.max_draft_len,
|
||||
use_mrope=self.use_mrope)
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(request_ids=list(
|
||||
range(num_ctx_requests, num_ctx_requests +
|
||||
num_gen_tokens)))
|
||||
|
||||
result = ScheduledRequests()
|
||||
result.context_requests = ctx_requests
|
||||
result.generation_requests = gen_requests
|
||||
|
||||
return result
|
||||
def _run_torch_compile_warmup(self, resource_manager: ResourceManager):
|
||||
"""Runs warmup iterations to specialize torch.compile kernels."""
|
||||
if not self._torch_compile_enabled:
|
||||
return
|
||||
|
||||
logger.info("Running torch.compile warmup...")
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
curr_max_num_tokens = min(
|
||||
kv_cache_manager.get_num_available_tokens(
|
||||
self.original_max_draft_len), self.max_num_tokens,
|
||||
self.batch_size * (self.max_seq_len - 1))
|
||||
|
||||
def get_autotune_warmup_request():
|
||||
return get_warmup_request(curr_max_num_tokens, 0)
|
||||
warmup_requests_configs = {
|
||||
(1, 1), # Specialize for 1 token.
|
||||
(self.batch_size,
|
||||
self.batch_size), # max_batch_size, pure generation
|
||||
(2, 0), # Non-one, pure context
|
||||
(curr_max_num_tokens, 0), # max_num_tokens, pure context
|
||||
}
|
||||
|
||||
@contextlib.contextmanager
|
||||
def release_batch(result: ScheduledRequests | None):
|
||||
try:
|
||||
yield result
|
||||
finally:
|
||||
if result is not None:
|
||||
for req in result.all_requests():
|
||||
kv_cache_manager.free_resources(req)
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.free_resources(req)
|
||||
# Disable cuda graph capture here so that we can properly capture it later
|
||||
with self.no_cuda_graph():
|
||||
for num_tokens, num_gen_tokens in warmup_requests_configs:
|
||||
with self._release_batch_context(
|
||||
self._create_warmup_request(resource_manager,
|
||||
num_tokens, num_gen_tokens),
|
||||
resource_manager) as batch:
|
||||
if batch is None:
|
||||
continue # Not enough KV cache space
|
||||
logger.info(
|
||||
f"Run warmup with {num_tokens} tokens, include {num_gen_tokens} generation tokens"
|
||||
)
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
# TODO: current warmup_request is not suitable for star attention
|
||||
cp_type = self.mapping.cp_config.get('cp_type', None)
|
||||
if cp_type == CpType.STAR:
|
||||
def _run_autotuner_warmup(self, resource_manager: ResourceManager):
|
||||
"""Runs a forward pass to populate the autotuner cache."""
|
||||
if not self.pytorch_backend_config.enable_autotuner:
|
||||
return
|
||||
|
||||
if self._torch_compile_enabled:
|
||||
logger.info("Running autotuner warmup...")
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
curr_max_num_tokens = min(
|
||||
kv_cache_manager.get_num_available_tokens(
|
||||
self.original_max_draft_len), self.max_num_tokens,
|
||||
self.batch_size * (self.max_seq_len - 1))
|
||||
|
||||
warmup_requests = set([
|
||||
(1, 1), # Specialize for 1 token.
|
||||
(self.batch_size,
|
||||
self.batch_size), # max_batch_size, pure generation
|
||||
(2, 0), # Non-one, pure context
|
||||
(curr_max_num_tokens, 0), # max_num_tokens, pure context
|
||||
])
|
||||
cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH", None)
|
||||
with self.no_cuda_graph(), autotune(cache_path=cache_path,
|
||||
rank=self.mapping.rank):
|
||||
warmup_request = self._create_warmup_request(
|
||||
resource_manager, curr_max_num_tokens, 0)
|
||||
with self._release_batch_context(warmup_request,
|
||||
resource_manager) as batch:
|
||||
if batch is not None:
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
# Disable cuda graph capture here so that we can properly capture it later
|
||||
with self.no_cuda_graph():
|
||||
for warmup_num_tokens, warmup_num_gen_tokens in warmup_requests:
|
||||
|
||||
with release_batch(
|
||||
get_warmup_request(warmup_num_tokens,
|
||||
warmup_num_gen_tokens)) as batch:
|
||||
if batch is None:
|
||||
# No KV cache space!
|
||||
continue
|
||||
logger.info(
|
||||
f"Run warmup with {warmup_num_tokens} tokens, include {warmup_num_gen_tokens} generation tokens"
|
||||
)
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
if self.pytorch_backend_config.enable_autotuner:
|
||||
# handle multiple rank issue
|
||||
cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH", None)
|
||||
with self.no_cuda_graph(), autotune(cache_path=cache_path,
|
||||
rank=self.mapping.rank):
|
||||
result = get_autotune_warmup_request()
|
||||
with release_batch(result) as batch:
|
||||
if batch is None:
|
||||
# No KV cache space!
|
||||
pass
|
||||
else:
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
logger.info(
|
||||
f"[Autotuner] Cache size after warmup is {len(AutoTuner.get().profiling_cache)}"
|
||||
)
|
||||
|
||||
AutoTuner.get().print_profiling_cache()
|
||||
logger.info(
|
||||
f"[Autotuner] Cache size after warmup is {len(AutoTuner.get().profiling_cache)}"
|
||||
)
|
||||
AutoTuner.get().print_profiling_cache()
|
||||
|
||||
def _run_cuda_graph_warmup(self, resource_manager: ResourceManager):
|
||||
"""Captures CUDA graphs for various batch sizes and draft lengths."""
|
||||
if not (self.cuda_graph_runner.enabled
|
||||
or self._torch_compile_piecewise_cuda_graph):
|
||||
return
|
||||
|
||||
self._capture_generation_cuda_graphs(resource_manager)
|
||||
self._capture_piecewise_cuda_graphs(resource_manager)
|
||||
|
||||
def _capture_generation_cuda_graphs(self,
|
||||
resource_manager: ResourceManager):
|
||||
"""Captures CUDA graphs for pure generation steps."""
|
||||
if not self.cuda_graph_runner.enabled:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Creating CUDA graph instances for {len(self._cuda_graph_batch_sizes)} batch sizes."
|
||||
)
|
||||
# Reverse the order of the cuda graph batch sizes to make smaller batch size graph could reuse larger batch size graph memory
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
|
||||
# Reverse order so smaller graphs can reuse memory from larger ones
|
||||
cuda_graph_batch_sizes = sorted(self._cuda_graph_batch_sizes,
|
||||
reverse=True)
|
||||
# Create CUDA graphs for different draft lengths
|
||||
draft_lengths = [self.max_draft_len]
|
||||
# For non-draft model, we also capture the CUDA graph instance for draft length 0,
|
||||
# so that when we disable spec decode at runtime, we can still run the captured graph.
|
||||
# Note that for one engine mode, we are not able to turn off spec decode at runtime.
|
||||
if (not self.is_draft_model and self.max_draft_len > 0
|
||||
and not self.spec_config.spec_dec_mode.use_one_engine()
|
||||
# Assume that speculation is always on if the user didn't give us a max_concurrency
|
||||
# value. This will save on memory.
|
||||
and self.spec_config.max_concurrency is not None):
|
||||
draft_lengths.append(0)
|
||||
if self.is_spec_decode and self.is_draft_model and spec_resource_manager is not None and isinstance(
|
||||
spec_resource_manager, Eagle3ResourceManager):
|
||||
draft_lengths.append(self.original_max_draft_len)
|
||||
draft_lengths = []
|
||||
if self.is_draft_model:
|
||||
if self.model_is_wrapped and self.is_spec_decode and spec_resource_manager is not None and isinstance(
|
||||
spec_resource_manager, Eagle3ResourceManager):
|
||||
# The CDL path uses draft_len > 0 for the number of iterations in the drafting loop.
|
||||
draft_lengths.append(self.original_max_draft_len)
|
||||
else:
|
||||
draft_lengths.append(self.max_draft_len)
|
||||
else:
|
||||
# For non-draft model, we also capture the CUDA graph instance for draft length 0,
|
||||
# so that when we disable spec decode at runtime, we can still run the captured graph.
|
||||
# Note that for one engine mode, we are not able to turn off spec decode at runtime.
|
||||
if (self.max_draft_len > 0
|
||||
and not self.spec_config.spec_dec_mode.use_one_engine()
|
||||
# Assume that speculation is always on if the user didn't give us a max_concurrency
|
||||
# value. This will save on memory.
|
||||
and self.spec_config.max_concurrency is not None):
|
||||
draft_lengths.append(0)
|
||||
draft_lengths = [self.max_draft_len]
|
||||
|
||||
for bs in cuda_graph_batch_sizes:
|
||||
if bs > self.batch_size:
|
||||
# skip batch size larger than self.batch_size
|
||||
continue
|
||||
|
||||
for draft_len in draft_lengths:
|
||||
with release_batch(get_cuda_graph_warmup_request(
|
||||
bs, draft_len)) as batch:
|
||||
warmup_request = self._create_cuda_graph_warmup_request(
|
||||
resource_manager, bs, draft_len)
|
||||
with self._release_batch_context(warmup_request,
|
||||
resource_manager) as batch:
|
||||
if batch is None:
|
||||
# No KV cache space!
|
||||
# No KV cache space, cannot continue capturing graphs
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Run generation only CUDA graph warmup for batch size={bs}, draft_len={draft_len}"
|
||||
f"Run generation-only CUDA graph warmup for batch size={bs}, draft_len={draft_len}"
|
||||
)
|
||||
|
||||
self.enable_spec_decode = draft_len > 0 or self.is_draft_model
|
||||
|
||||
def _update_draft_inference_state(is_first_draft: bool,
|
||||
batch: ScheduledRequests):
|
||||
if self.is_draft_model and isinstance(
|
||||
spec_resource_manager, Eagle3ResourceManager):
|
||||
spec_resource_manager.is_first_draft = is_first_draft
|
||||
if is_first_draft:
|
||||
for req in batch.generation_requests:
|
||||
req.py_is_first_draft = True
|
||||
# Reset the draft tokens for the first draft inference
|
||||
req.py_draft_tokens = []
|
||||
|
||||
_update_draft_inference_state(draft_len > 0, batch)
|
||||
self._update_draft_inference_state_for_warmup(
|
||||
batch, draft_len > 0, resource_manager)
|
||||
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
if self._torch_compile_piecewise_cuda_graph and self._torch_compile_enabled:
|
||||
piecewise_cuda_graph_num_tokens = sorted(
|
||||
self._piecewise_cuda_graph_num_tokens, reverse=True)
|
||||
def _capture_piecewise_cuda_graphs(self, resource_manager: ResourceManager):
|
||||
"""Captures piecewise CUDA graphs for context/prefill steps via torch.compile."""
|
||||
if not (self._torch_compile_piecewise_cuda_graph
|
||||
and self._torch_compile_enabled):
|
||||
return
|
||||
|
||||
with capture_piecewise_cuda_graph(True):
|
||||
for num_tokens in piecewise_cuda_graph_num_tokens:
|
||||
with self.no_cuda_graph():
|
||||
with release_batch(get_warmup_request(num_tokens,
|
||||
0)) as batch:
|
||||
logger.info(
|
||||
f"Run piecewise CUDA graph warmup for num tokens={num_tokens}"
|
||||
)
|
||||
logger.info("Running piecewise CUDA graph warmup...")
|
||||
piecewise_cuda_graph_num_tokens = sorted(
|
||||
self._piecewise_cuda_graph_num_tokens, reverse=True)
|
||||
|
||||
for _ in range(3):
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
with capture_piecewise_cuda_graph(True), self.no_cuda_graph():
|
||||
for num_tokens in piecewise_cuda_graph_num_tokens:
|
||||
warmup_request = self._create_warmup_request(
|
||||
resource_manager, num_tokens, 0)
|
||||
with self._release_batch_context(warmup_request,
|
||||
resource_manager) as batch:
|
||||
if batch is None:
|
||||
continue
|
||||
|
||||
# Set the value back to the original value
|
||||
self.enable_spec_decode = self.is_spec_decode
|
||||
logger.info(
|
||||
f"Run piecewise CUDA graph warmup for num tokens={num_tokens}"
|
||||
)
|
||||
# Run a few times to ensure capture
|
||||
for _ in range(3):
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
### Helper methods promoted from the original warmup method ###
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _release_batch_context(self, batch: Optional[ScheduledRequests],
|
||||
resource_manager: ResourceManager):
|
||||
"""A context manager to automatically free resources of a dummy batch."""
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
try:
|
||||
yield batch
|
||||
finally:
|
||||
if batch is not None and kv_cache_manager is not None:
|
||||
for req in batch.all_requests():
|
||||
kv_cache_manager.free_resources(req)
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.free_resources(req)
|
||||
|
||||
def _get_num_extra_decoding_steps(self) -> int:
|
||||
"""Determines extra decoding steps needed for fused drafting loops."""
|
||||
if isinstance(self.model, ChainDrafter):
|
||||
return self.model.max_draft_len
|
||||
else:
|
||||
assert not self.model_is_wrapped, (
|
||||
f"Please add logic to determine num_extra_decoding_steps for drafting loop {type(self.model)}"
|
||||
)
|
||||
return 0
|
||||
|
||||
def _create_warmup_request(
|
||||
self, resource_manager: ResourceManager, num_tokens: int,
|
||||
num_gen_tokens: int) -> Optional[ScheduledRequests]:
|
||||
"""Creates a generic dummy ScheduledRequests object for warmup."""
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
|
||||
available_tokens = kv_cache_manager.get_num_available_tokens(
|
||||
self.runtime_draft_len)
|
||||
available_blocks = kv_cache_manager.get_num_free_blocks()
|
||||
if num_tokens > self.max_num_tokens or num_tokens > available_tokens:
|
||||
return None
|
||||
|
||||
num_extra_decoding_steps = self._get_num_extra_decoding_steps()
|
||||
if num_extra_decoding_steps > 0:
|
||||
return None # Disable autotuning for fused drafting loops for now.
|
||||
|
||||
num_ctx_tokens = num_tokens - num_gen_tokens
|
||||
num_ctx_requests = 0
|
||||
ctx_requests = []
|
||||
gen_requests = []
|
||||
|
||||
max_seq_len = self.max_seq_len - 1
|
||||
num_full_seqs = 0
|
||||
num_left_over_tokens = 0
|
||||
|
||||
if num_ctx_tokens > 0:
|
||||
num_full_seqs = num_ctx_tokens // max_seq_len
|
||||
num_left_over_tokens = num_ctx_tokens - num_full_seqs * max_seq_len
|
||||
num_ctx_requests = num_full_seqs + (1 if num_left_over_tokens > 0
|
||||
else 0)
|
||||
|
||||
if num_ctx_requests + num_gen_tokens > self.batch_size:
|
||||
return None # Not enough batch size to fill the request
|
||||
|
||||
blocks_to_use = num_full_seqs * math.ceil(
|
||||
max_seq_len / kv_cache_manager.tokens_per_block) + math.ceil(
|
||||
num_left_over_tokens /
|
||||
kv_cache_manager.tokens_per_block) + num_gen_tokens
|
||||
|
||||
if blocks_to_use > available_blocks:
|
||||
return None
|
||||
|
||||
if num_ctx_tokens > 0:
|
||||
ctx_token_nums = [max_seq_len] * num_full_seqs
|
||||
if num_left_over_tokens > 0:
|
||||
ctx_token_nums.append(num_left_over_tokens)
|
||||
|
||||
ctx_requests = kv_cache_manager.add_dummy_requests(
|
||||
list(range(num_ctx_requests)),
|
||||
token_nums=ctx_token_nums,
|
||||
is_gen=False,
|
||||
max_num_draft_tokens=self.runtime_draft_len,
|
||||
use_mrope=self.use_mrope)
|
||||
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(
|
||||
request_ids=list(range(num_ctx_requests)))
|
||||
|
||||
if num_gen_tokens > 0:
|
||||
gen_requests = kv_cache_manager.add_dummy_requests(
|
||||
list(range(num_ctx_requests,
|
||||
num_ctx_requests + num_gen_tokens)),
|
||||
token_nums=[1] * num_gen_tokens,
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=self.max_draft_len,
|
||||
use_mrope=self.use_mrope)
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(request_ids=list(
|
||||
range(num_ctx_requests, num_ctx_requests + num_gen_tokens)))
|
||||
|
||||
result = ScheduledRequests()
|
||||
result.context_requests = ctx_requests
|
||||
result.generation_requests = gen_requests
|
||||
return result
|
||||
|
||||
def _create_cuda_graph_warmup_request(
|
||||
self, resource_manager: ResourceManager, batch_size: int,
|
||||
draft_len: int) -> Optional[ScheduledRequests]:
|
||||
"""Creates a dummy ScheduledRequests tailored for CUDA graph capture."""
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
|
||||
available_blocks = kv_cache_manager.get_num_free_blocks(
|
||||
) // self.max_beam_width
|
||||
if available_blocks < batch_size:
|
||||
return None
|
||||
|
||||
result = ScheduledRequests()
|
||||
result.context_requests = []
|
||||
num_extra_decoding_steps = self._get_num_extra_decoding_steps()
|
||||
|
||||
# Add (batch_size - 1) dummy requests with seq_len=1.
|
||||
requests = kv_cache_manager.add_dummy_requests(
|
||||
list(range(batch_size - 1)),
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=draft_len,
|
||||
use_mrope=self.use_mrope,
|
||||
max_beam_width=self.max_beam_width,
|
||||
num_extra_decoding_steps=num_extra_decoding_steps)
|
||||
|
||||
available_tokens = kv_cache_manager.get_num_available_tokens(draft_len)
|
||||
|
||||
# Add one dummy request with the maximum possible sequence length.
|
||||
token_num = max(1, min(available_tokens, self.max_seq_len - 1))
|
||||
model_config = self.model.model_config.pretrained_config
|
||||
max_position_embeddings = getattr(model_config,
|
||||
'max_position_embeddings', None)
|
||||
if max_position_embeddings is not None:
|
||||
token_num = min(token_num, max_position_embeddings - draft_len)
|
||||
|
||||
assert token_num > num_extra_decoding_steps, (
|
||||
"Cannot fuse drafting loop. Not enough KV cache space for all draft tokens."
|
||||
)
|
||||
token_num -= num_extra_decoding_steps
|
||||
|
||||
max_seq_len_request = kv_cache_manager.add_dummy_requests(
|
||||
request_ids=[batch_size - 1],
|
||||
token_nums=[token_num],
|
||||
is_gen=True,
|
||||
max_num_draft_tokens=draft_len,
|
||||
use_mrope=self.use_mrope,
|
||||
max_beam_width=self.max_beam_width,
|
||||
num_extra_decoding_steps=num_extra_decoding_steps)[0]
|
||||
|
||||
# Insert the longest request first to simulate padding for the CUDA graph.
|
||||
requests.insert(0, max_seq_len_request)
|
||||
result.generation_requests = requests
|
||||
if spec_resource_manager is not None:
|
||||
spec_resource_manager.add_dummy_requests(
|
||||
request_ids=list(range(batch_size)))
|
||||
return result
|
||||
|
||||
def _get_cuda_graph_draft_lengths(
|
||||
self, resource_manager: ResourceManager) -> List[int]:
|
||||
"""Determines the draft lengths for which to capture CUDA graphs."""
|
||||
draft_lengths = [self.max_draft_len]
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
|
||||
# For non-draft model, also capture a graph for draft_len=0
|
||||
if (not self.is_draft_model and self.max_draft_len > 0
|
||||
and not self.spec_config.spec_dec_mode.use_one_engine()
|
||||
and self.spec_config.max_concurrency is not None):
|
||||
draft_lengths.append(0)
|
||||
|
||||
# Special case for Eagle3 draft model
|
||||
if (self.is_spec_decode and self.is_draft_model
|
||||
and isinstance(spec_resource_manager, Eagle3ResourceManager)):
|
||||
draft_lengths.append(self.original_max_draft_len)
|
||||
|
||||
return list(set(draft_lengths)) # Use set to remove duplicates
|
||||
|
||||
def _update_draft_inference_state_for_warmup(
|
||||
self, batch: ScheduledRequests, is_first_draft: bool,
|
||||
resource_manager: ResourceManager):
|
||||
"""Updates request states for specific draft model warmups like Eagle3."""
|
||||
spec_resource_manager = resource_manager.get_resource_manager(
|
||||
ResourceManagerType.SPEC_RESOURCE_MANAGER)
|
||||
if self.is_draft_model and isinstance(spec_resource_manager,
|
||||
Eagle3ResourceManager):
|
||||
spec_resource_manager.is_first_draft = is_first_draft
|
||||
if is_first_draft:
|
||||
for req in batch.generation_requests:
|
||||
req.py_is_first_draft = True
|
||||
req.py_draft_tokens = []
|
||||
|
||||
def _set_up_attn_metadata(self, kv_cache_manager: KVCacheManager):
|
||||
enable_context_mla_with_cached_kv = is_mla(
|
||||
@ -787,7 +878,8 @@ class PyTorchModelEngine(ModelEngine):
|
||||
enable_flash_mla=self.model.model_config.enable_flash_mla,
|
||||
enable_context_mla_with_cached_kv=
|
||||
enable_context_mla_with_cached_kv,
|
||||
cache_indirection=cache_indirection)
|
||||
cache_indirection=cache_indirection,
|
||||
sparse_attention_config=self.sparse_attention_config)
|
||||
|
||||
if self.attn_metadata is not None:
|
||||
# This assertion can be relaxed if needed: just create a new metadata
|
||||
@ -804,7 +896,8 @@ class PyTorchModelEngine(ModelEngine):
|
||||
runtime_features=self.attn_runtime_features,
|
||||
enable_flash_mla=self.model.model_config.enable_flash_mla,
|
||||
enable_context_mla_with_cached_kv=enable_context_mla_with_cached_kv,
|
||||
cache_indirection=cache_indirection)
|
||||
cache_indirection=cache_indirection,
|
||||
sparse_attention_config=self.sparse_attention_config)
|
||||
|
||||
return self.attn_metadata
|
||||
|
||||
@ -1139,6 +1232,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
prompt_lengths.append(len(prompt_tokens))
|
||||
past_seen_token_num = begin_compute
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
|
||||
# Multimodal
|
||||
py_multimodal_runtime = MultimodalRuntimeData(
|
||||
@ -1249,6 +1343,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
range(past_seen_token_num,
|
||||
past_seen_token_num + 1 + num_draft_tokens)))
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
# update batch index
|
||||
request.py_batch_idx = request.py_seq_slot
|
||||
else:
|
||||
@ -1282,6 +1377,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
else:
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num +
|
||||
self.runtime_draft_len + 1)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
if self.enable_spec_decode and spec_config.spec_dec_mode.extend_ctx(
|
||||
self.attn_backend):
|
||||
prompt_lengths.append(1 + self.runtime_draft_len)
|
||||
@ -1334,8 +1430,15 @@ class PyTorchModelEngine(ModelEngine):
|
||||
if beam == first_beam:
|
||||
previous_batch_indices.append(request.py_batch_idx)
|
||||
past_seen_token_num = request.max_beam_num_tokens
|
||||
position_ids.append(past_seen_token_num)
|
||||
position_id = past_seen_token_num
|
||||
if self.mapping.has_cp_helix():
|
||||
# Do an allgather among CP ranks to get the complete sequence length seen by all CP ranks.
|
||||
past_seen_token_nums = self.dist.cp_allgather(
|
||||
past_seen_token_num)
|
||||
position_id = sum(past_seen_token_nums)
|
||||
position_ids.append(position_id)
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
prompt_lengths.append(request.py_prompt_len)
|
||||
draft_lens.append(0)
|
||||
sequence_lengths.append(1)
|
||||
@ -1858,6 +1961,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
sequence_lengths.append(len(input_id))
|
||||
block_ids_per_seq.extend([all_cache_indices])
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
num_contexts = len(sequence_lengths)
|
||||
for request in scheduled_requests.context_requests:
|
||||
ctx_iter = request.ctx_iters
|
||||
@ -1897,6 +2001,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
sequence_lengths.append(len(input_id))
|
||||
block_ids_per_seq.extend([all_cache_indices])
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
num_queries = len(sequence_lengths) - num_contexts
|
||||
|
||||
# Requests with draft tokens are treated like extend requests.
|
||||
@ -1954,6 +2059,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
position_ids.append(last_query_pos_id + request.gen_iters + 1)
|
||||
block_ids_per_seq.extend([all_cache_indices])
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
request.cached_tokens = num_cached_tokens_per_seq[-1]
|
||||
|
||||
num_tokens = len(input_ids)
|
||||
assert num_tokens <= self.max_num_tokens, (
|
||||
@ -2111,13 +2217,17 @@ class PyTorchModelEngine(ModelEngine):
|
||||
if CpType.STAR == cp_type:
|
||||
return self._prepare_star_attention_inputs(
|
||||
scheduled_requests, kv_cache_manager, attn_metadata)
|
||||
elif CpType.HELIX == cp_type:
|
||||
# Take the usual route of _prepare_tp_inputs.
|
||||
pass
|
||||
else:
|
||||
assert False, f'Unsupport cp_type {cp_type}'
|
||||
else:
|
||||
return self._prepare_tp_inputs(scheduled_requests, kv_cache_manager,
|
||||
attn_metadata, spec_metadata,
|
||||
new_tensors_device,
|
||||
cache_indirection_buffer)
|
||||
raise NotImplementedError(
|
||||
f"Unsupported cp_type {getattr(cp_type, 'name', cp_type)}.")
|
||||
|
||||
return self._prepare_tp_inputs(scheduled_requests, kv_cache_manager,
|
||||
attn_metadata, spec_metadata,
|
||||
new_tensors_device,
|
||||
cache_indirection_buffer)
|
||||
|
||||
@torch.inference_mode()
|
||||
@with_model_extra_attrs(lambda self: self.model.extra_attrs)
|
||||
@ -2186,35 +2296,38 @@ class PyTorchModelEngine(ModelEngine):
|
||||
new_tensors_device, cache_indirection_buffer)
|
||||
|
||||
self.iter_counter += 1
|
||||
|
||||
if not maybe_graph:
|
||||
# Fallback to eager execution if graph was not used
|
||||
with MoeLoadBalancerIterContext(moe_load_balancer):
|
||||
outputs = self._forward_step(inputs, gather_ids,
|
||||
gather_context_logits)
|
||||
else:
|
||||
if self.cuda_graph_runner.needs_capture(key):
|
||||
|
||||
def capture_forward_fn(inputs: Dict[str, Any]):
|
||||
with MoeLoadBalancerIterContext(moe_load_balancer):
|
||||
return self._forward_step(
|
||||
inputs,
|
||||
gather_ids=gather_ids,
|
||||
gather_context_logits=gather_context_logits)
|
||||
|
||||
def capture_postprocess_fn(inputs: Dict[str, Any]):
|
||||
self._postprocess_inputs(inputs)
|
||||
|
||||
self.cuda_graph_runner.capture(key, capture_forward_fn,
|
||||
inputs,
|
||||
capture_postprocess_fn)
|
||||
|
||||
# here we don't need to use context since cuda graph capture didn't run kernel.
|
||||
# maybe we need a cleaner way to do this.
|
||||
outputs = self.cuda_graph_runner.replay(key, inputs)
|
||||
else:
|
||||
with with_shared_pool(self.cuda_graph_runner.get_graph_pool()):
|
||||
if not maybe_graph:
|
||||
# Fallback to eager execution if graph was not used
|
||||
with MoeLoadBalancerIterContext(moe_load_balancer):
|
||||
outputs = self._forward_step(inputs, gather_ids,
|
||||
gather_context_logits)
|
||||
else:
|
||||
if self.cuda_graph_runner.needs_capture(key):
|
||||
|
||||
def capture_forward_fn(inputs: Dict[str, Any]):
|
||||
with MoeLoadBalancerIterContext(moe_load_balancer):
|
||||
return self._forward_step(
|
||||
inputs,
|
||||
gather_ids=gather_ids,
|
||||
gather_context_logits=gather_context_logits)
|
||||
|
||||
def capture_postprocess_fn(inputs: Dict[str, Any]):
|
||||
self._postprocess_inputs(inputs)
|
||||
|
||||
self.cuda_graph_runner.capture(key, capture_forward_fn,
|
||||
inputs,
|
||||
capture_postprocess_fn)
|
||||
|
||||
# here we don't need to use context since cuda graph capture didn't run kernel.
|
||||
# maybe we need a cleaner way to do this.
|
||||
outputs = self.cuda_graph_runner.replay(key, inputs)
|
||||
else:
|
||||
with MoeLoadBalancerIterContext(moe_load_balancer):
|
||||
outputs = self.cuda_graph_runner.replay(key, inputs)
|
||||
|
||||
if self.forward_pass_callable is not None:
|
||||
self.forward_pass_callable()
|
||||
|
||||
self._execute_logit_post_processors(scheduled_requests, outputs)
|
||||
|
||||
@ -2247,21 +2360,34 @@ class PyTorchModelEngine(ModelEngine):
|
||||
inputs = self._preprocess_inputs(inputs)
|
||||
if inputs.get('spec_metadata', None):
|
||||
gather_ids = inputs['spec_metadata'].gather_ids
|
||||
if self.without_logits:
|
||||
outputs = self.model_forward(**inputs)
|
||||
return outputs
|
||||
|
||||
# For simplicity, just return all the the logits if we have special gather_ids
|
||||
# from speculative decoding.
|
||||
logits = self.model_forward(
|
||||
outputs = self.model_forward(
|
||||
**inputs,
|
||||
return_context_logits=gather_ids is not None
|
||||
or gather_context_logits,
|
||||
)
|
||||
if gather_ids is not None:
|
||||
return {'logits': logits[gather_ids]}
|
||||
|
||||
if self.without_logits:
|
||||
return outputs
|
||||
|
||||
if isinstance(outputs, dict):
|
||||
# If the model returns a dict, get the logits from it. All other keys are kept.
|
||||
logits = outputs.get('logits', None)
|
||||
# If the logits are not found, no further processing is needed.
|
||||
if logits is None:
|
||||
return outputs
|
||||
else:
|
||||
return {'logits': logits}
|
||||
# If the model returns a single tensor, assume it is the logits and wrap it in a dict.
|
||||
logits = outputs
|
||||
outputs = {'logits': logits}
|
||||
|
||||
# If we have special gather_ids, gather the logits
|
||||
if gather_ids is not None:
|
||||
outputs['logits'] = logits[gather_ids]
|
||||
|
||||
return outputs
|
||||
|
||||
@nvtx_range("_forward_step_mm_encoder_only")
|
||||
def _forward_step_mm_encoder_only(
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -678,9 +682,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -599,7 +603,7 @@
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">'plugin_config'</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plugin_config</span><span class="p">,</span> <span class="n">PluginConfig</span><span class="p">),</span> \
|
||||
<span class="sa">f</span><span class="s2">"Found unexpected plugin_config object with type: </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">plugin_config</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="n">config</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_config</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
|
||||
<span class="n">config</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">plugin_config</span><span class="o">.</span><span class="n">model_dump</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s2">"json"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">config</span>
|
||||
|
||||
|
||||
@ -1180,7 +1184,8 @@
|
||||
<span class="k">if</span> <span class="n">plugin_config</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">plugin_config</span> <span class="o">=</span> <span class="n">PluginConfig</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="s2">"plugin_config"</span> <span class="ow">in</span> <span class="n">config</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||
<span class="n">plugin_config</span><span class="o">.</span><span class="n">update_from_dict</span><span class="p">(</span><span class="n">config</span><span class="p">[</span><span class="s2">"plugin_config"</span><span class="p">])</span>
|
||||
<span class="n">plugin_config</span> <span class="o">=</span> <span class="n">plugin_config</span><span class="o">.</span><span class="n">model_copy</span><span class="p">(</span>
|
||||
<span class="n">update</span><span class="o">=</span><span class="n">config</span><span class="p">[</span><span class="s2">"plugin_config"</span><span class="p">],</span> <span class="n">deep</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="n">dry_run</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'dry_run'</span><span class="p">,</span> <span class="n">defaults</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'dry_run'</span><span class="p">))</span>
|
||||
<span class="n">visualize_network</span> <span class="o">=</span> <span class="n">config</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'visualize_network'</span><span class="p">,</span>
|
||||
@ -1239,7 +1244,7 @@
|
||||
<span class="c1"># the enum KVCacheType cannot be converted automatically</span>
|
||||
<span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'kv_cache_type'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">output</span><span class="p">[</span><span class="s1">'kv_cache_type'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">output</span><span class="p">[</span><span class="s1">'kv_cache_type'</span><span class="p">]</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||
<span class="n">output</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
|
||||
<span class="n">output</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="s1">'plugin_config'</span><span class="p">]</span><span class="o">.</span><span class="n">model_dump</span><span class="p">()</span>
|
||||
<span class="n">output</span><span class="p">[</span><span class="s1">'lora_config'</span><span class="p">]</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="s1">'lora_config'</span><span class="p">]</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
|
||||
<span class="n">output</span><span class="p">[</span><span class="s1">'auto_parallel_config'</span><span class="p">]</span> <span class="o">=</span> <span class="n">output</span><span class="p">[</span><span class="s1">'auto_parallel_config'</span><span class="p">]</span><span class="o">.</span><span class="n">to_dict</span><span class="p">(</span>
|
||||
<span class="p">)</span>
|
||||
@ -2064,9 +2069,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -708,9 +712,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -750,9 +754,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -498,6 +502,7 @@
|
||||
<h1>Source code for tensorrt_llm.executor.result</h1><div class="highlight"><pre>
|
||||
<span></span><span class="kn">import</span><span class="w"> </span><span class="nn">asyncio</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">threading</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">weakref</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span><span class="p">,</span> <span class="n">field</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">queue</span><span class="w"> </span><span class="kn">import</span> <span class="n">Empty</span><span class="p">,</span> <span class="n">Queue</span>
|
||||
@ -508,11 +513,17 @@
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">torch</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">torch.nn.functional</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">F</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">nvtx_range_debug</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">ray</span>
|
||||
<span class="k">except</span> <span class="ne">ModuleNotFoundError</span><span class="p">:</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm</span><span class="w"> </span><span class="kn">import</span> <span class="n">ray_stub</span> <span class="k">as</span> <span class="n">ray</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._ray_utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">unwrap_ray_errors</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">mpi_disabled</span><span class="p">,</span> <span class="n">nvtx_range_debug</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..bindings</span><span class="w"> </span><span class="kn">import</span> <span class="n">executor</span> <span class="k">as</span> <span class="n">tllm</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..disaggregated_params</span><span class="w"> </span><span class="kn">import</span> <span class="n">DisaggregatedParams</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..llmapi.tracer</span><span class="w"> </span><span class="kn">import</span> <span class="n">global_tracer</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..llmapi.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">AsyncQueue</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..llmapi.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">AsyncQueue</span><span class="p">,</span> <span class="n">print_traceback_on_error</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..metrics</span><span class="w"> </span><span class="kn">import</span> <span class="n">MetricNames</span><span class="p">,</span> <span class="n">MetricsCollector</span><span class="p">,</span> <span class="n">RequestEventTiming</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..sampling_params</span><span class="w"> </span><span class="kn">import</span> <span class="n">LogprobParams</span><span class="p">,</span> <span class="n">SamplingParams</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">ErrorResponse</span><span class="p">,</span> <span class="n">has_event_loop</span><span class="p">,</span> <span class="n">is_llm_response</span>
|
||||
@ -596,6 +607,8 @@
|
||||
<span class="sd"> finish_reason (Literal['stop', 'length', 'timeout', 'cancelled'], optional): The reason why the sequence is finished. Defaults to None.</span>
|
||||
<span class="sd"> stop_reason (int, str, optional): The stop string or token id that caused the completion to stop, None if the completion finished for some other reason. Defaults to None.</span>
|
||||
<span class="sd"> generation_logits (torch.Tensor, optional): The logits on the generated output token ids. Defaults to None.</span>
|
||||
<span class="sd"> additional_context_outputs (Dict[str, torch.Tensor], optional): The additional context outputs. Defaults to None.</span>
|
||||
<span class="sd"> additional_generation_outputs (Dict[str, torch.Tensor], optional): The additional generation outputs. Defaults to None.</span>
|
||||
<span class="sd"> disaggregated_params (tensorrt_llm.disaggregated_params.DisaggregatedParams, optional): Parameters needed for disaggregated serving. Includes the type of request, the first generated tokens, the context request id and the any additional state needing to be transferred from context and generation instances. Defaults to None.</span>
|
||||
<span class="sd"> request_perf_metrics (tensorrt_llm.bindings.executor.RequestPerfMetrics, optional): Performance metrics for the request. Defaults to None.</span>
|
||||
|
||||
@ -616,6 +629,8 @@
|
||||
<span class="s1">'cancelled'</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">stop_reason</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">generation_logits</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">additional_context_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">additional_generation_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">disaggregated_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DisaggregatedParams</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">request_perf_metrics</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">tllm</span><span class="o">.</span><span class="n">RequestPerfMetrics</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
@ -647,12 +662,104 @@
|
||||
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">warmup_tensorrt_llm</span><span class="p">():</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tensorrt_llm</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Warmup by importing tensorrt_llm with version"</span><span class="p">,</span>
|
||||
<span class="n">tensorrt_llm</span><span class="o">.</span><span class="n">version</span><span class="o">.</span><span class="n">__version__</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="nd">@ray</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="n">max_concurrency</span><span class="o">=</span><span class="mi">1000000</span><span class="p">,</span> <span class="n">num_cpus</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">RayAsyncQueue</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Ray actor for async response handling."""</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"Key </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> already registered"</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">unregister</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">warmup</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
<span class="n">warmup_tensorrt_llm</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">put_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">item</span><span class="p">:</span> <span class="n">Any</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"Key </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> not registered"</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
||||
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">get_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"Key </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> not registered"</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||
<span class="n">ret</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">ret</span>
|
||||
|
||||
|
||||
<span class="n">SYNC_QUEUE_MAX_CONCURRENCY</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
|
||||
|
||||
<span class="nd">@ray</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="n">max_concurrency</span><span class="o">=</span><span class="n">SYNC_QUEUE_MAX_CONCURRENCY</span><span class="p">,</span>
|
||||
<span class="n">num_cpus</span><span class="o">=</span><span class="n">SYNC_QUEUE_MAX_CONCURRENCY</span><span class="p">)</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">RaySyncQueue</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Ray actor for sync response handling."""</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">semaphore</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Semaphore</span><span class="p">(</span><span class="n">SYNC_QUEUE_MAX_CONCURRENCY</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">register</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">,</span> <span class="sa">f</span><span class="s2">"Key </span><span class="si">{</span><span class="n">key</span><span class="si">}</span><span class="s2"> already registered"</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">unregister</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">:</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">warmup</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span><span class="p">:</span>
|
||||
<span class="k">return</span>
|
||||
<span class="n">warmup_tensorrt_llm</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">warmup_done</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">put_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">item</span><span class="p">:</span> <span class="n">Any</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span>
|
||||
<span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">semaphore</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">event_map</span><span class="p">[</span><span class="n">key</span><span class="p">]</span><span class="o">.</span><span class="n">clear</span><span class="p">()</span>
|
||||
<span class="n">ret</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">ret</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">GenerationResultBase</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">''' This holds the core logic of the GenerationResult class. '''</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
|
||||
<span class="nb">id</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
|
||||
<span class="n">sampling_params</span><span class="p">:</span> <span class="n">SamplingParams</span><span class="p">,</span>
|
||||
<span class="n">ray_queue</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">RayAsyncQueue</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">background_error_handler</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Callable</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">postproc_params</span><span class="p">:</span> <span class="s2">"Optional[PostprocParams]"</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">id</span> <span class="o">=</span> <span class="nb">id</span>
|
||||
@ -660,18 +767,29 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">postproc_params</span> <span class="o">=</span> <span class="n">postproc_params</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">disaggregated_params</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">decoding_iter</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cached_tokens</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="c1"># Average decoded tokens per runtime iteration; set when the first LLM response arrives.</span>
|
||||
<span class="c1"># None indicates not yet available (e.g., before first step/stream).</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">avg_decoded_tokens_per_iter</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_done</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">metrics_dict</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">has_event_loop</span><span class="p">():</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="n">AsyncQueue</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span><span class="o">.</span><span class="n">sync_q</span>
|
||||
<span class="k">if</span> <span class="n">ray_queue</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">has_event_loop</span><span class="p">():</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="n">ray_queue</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">ray_queue</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">with</span> <span class="n">unwrap_ray_errors</span><span class="p">():</span>
|
||||
<span class="n">ray</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">register</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="nb">id</span><span class="p">))</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">has_event_loop</span><span class="p">():</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="n">AsyncQueue</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span><span class="o">.</span><span class="n">sync_q</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># In Sampling mode, the Executor runtime will return best_of sequences</span>
|
||||
<span class="c1"># in total, which the LLM API will select the n-best sequences among</span>
|
||||
@ -780,6 +898,14 @@
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">generation_logits</span> <span class="o">=</span> <span class="n">response_tensors</span><span class="o">.</span><span class="n">generation_logits</span><span class="p">[</span>
|
||||
<span class="n">src_idx</span><span class="p">,</span> <span class="p">:</span><span class="n">output</span><span class="o">.</span><span class="n">length</span><span class="p">]</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">response_tensors</span><span class="p">,</span> <span class="s1">'additional_context_outputs'</span><span class="p">,</span>
|
||||
<span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">additional_context_outputs</span> <span class="o">=</span> <span class="n">response_tensors</span><span class="o">.</span><span class="n">additional_context_outputs</span>
|
||||
|
||||
<span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">response_tensors</span><span class="p">,</span> <span class="s1">'additional_generation_outputs'</span><span class="p">,</span>
|
||||
<span class="kc">None</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">output</span><span class="o">.</span><span class="n">additional_generation_outputs</span> <span class="o">=</span> <span class="n">response_tensors</span><span class="o">.</span><span class="n">additional_generation_outputs</span>
|
||||
|
||||
<span class="c1"># when sampling_params.n > 1 and is cancelled, make sure all the outputs</span>
|
||||
<span class="c1"># be marked as cancelled.</span>
|
||||
<span class="k">if</span> <span class="n">finish_reasons</span> <span class="ow">and</span> <span class="n">finish_reasons</span><span class="p">[</span>
|
||||
@ -816,6 +942,7 @@
|
||||
<span class="sa">f</span><span class="s2">"Unknown finish reason: </span><span class="si">{</span><span class="n">finish_reasons</span><span class="p">[</span><span class="n">src_idx</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">record_stats</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">req_perf_metrics_dict</span><span class="p">)</span>
|
||||
|
||||
<span class="nd">@print_traceback_on_error</span>
|
||||
<span class="nd">@nvtx_range_debug</span><span class="p">(</span><span class="s2">"handle_response"</span><span class="p">,</span>
|
||||
<span class="n">color</span><span class="o">=</span><span class="s2">"red"</span><span class="p">,</span>
|
||||
<span class="n">category</span><span class="o">=</span><span class="s2">"GenerationResultBase"</span><span class="p">)</span>
|
||||
@ -837,6 +964,18 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">res</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">_postprocess_result</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">res</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_outputs</span><span class="p">[</span>
|
||||
<span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">request_perf_metrics</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">request_perf_metrics</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">disaggregated_params</span><span class="p">:</span>
|
||||
<span class="n">disaggregated_params</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">disaggregated_params</span>
|
||||
|
||||
<span class="c1"># Generation only response has no disaggregated_params attached</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">disaggregated_params</span><span class="p">:</span>
|
||||
<span class="n">disaggregated_params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">disaggregated_params</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_outputs</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">disaggregated_params</span> <span class="o">=</span> <span class="n">disaggregated_params</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">metrics</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">metrics_dict</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">metrics</span>
|
||||
|
||||
@ -858,6 +997,7 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_done</span> <span class="o">=</span> <span class="n">response_result</span><span class="o">.</span><span class="n">is_final</span>
|
||||
<span class="n">context_phase_params</span> <span class="o">=</span> <span class="n">response_result</span><span class="o">.</span><span class="n">context_phase_params</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">decoding_iter</span> <span class="o">=</span> <span class="n">response_result</span><span class="o">.</span><span class="n">decoding_iter</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cached_tokens</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">response_result</span><span class="p">,</span> <span class="s1">'cached_tokens'</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">avg_decoded_tokens_per_iter</span> <span class="o">=</span> <span class="n">response_result</span><span class="o">.</span><span class="n">avg_decoded_tokens_per_iter</span>
|
||||
<span class="k">if</span> <span class="n">context_phase_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">disaggregated_params</span> <span class="o">=</span> <span class="n">DisaggregatedParams</span><span class="p">(</span>
|
||||
@ -908,6 +1048,12 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Unknown response type: </span><span class="si">{</span><span class="n">response</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_done</span> <span class="ow">and</span> <span class="n">mpi_disabled</span><span class="p">():</span>
|
||||
<span class="k">assert</span> <span class="nb">hasattr</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="p">,</span> <span class="s2">"unregister"</span>
|
||||
<span class="p">),</span> <span class="s2">"Ray path should be activated for unregistering the Ray queue."</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">unregister</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">id</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">record_stats</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">output</span><span class="p">:</span> <span class="n">CompletionOutput</span><span class="p">,</span>
|
||||
<span class="n">stats</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
@ -1030,9 +1176,15 @@
|
||||
<span class="n">disaggregated_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DisaggregatedParams</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">logprob_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LogprobParams</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">use_async_queue</span> <span class="o">=</span> <span class="n">has_event_loop</span><span class="p">()</span>
|
||||
<span class="n">shared_queue</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">executor</span> <span class="ow">and</span> <span class="n">executor</span><span class="o">.</span><span class="n">use_ray_queue</span><span class="p">():</span>
|
||||
<span class="n">shared_queue</span> <span class="o">=</span> <span class="n">executor</span><span class="o">.</span><span class="n">async_response_queue_weakref</span> <span class="k">if</span> <span class="n">use_async_queue</span> <span class="k">else</span> <span class="n">executor</span><span class="o">.</span><span class="n">sync_response_queue_weakref</span>
|
||||
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="n">generation_request</span><span class="o">.</span><span class="n">id</span><span class="p">,</span>
|
||||
<span class="n">generation_request</span><span class="o">.</span><span class="n">sampling_params</span><span class="p">,</span>
|
||||
<span class="n">shared_queue</span><span class="p">,</span>
|
||||
<span class="n">background_error_handler</span><span class="p">,</span>
|
||||
<span class="n">postproc_params</span><span class="o">=</span><span class="n">generation_request</span><span class="o">.</span><span class="n">postproc_params</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
@ -1086,13 +1238,26 @@
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">"_logprob_params"</span><span class="p">):</span>
|
||||
<span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_logprob_params</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_handle_ray_response</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">response</span><span class="p">:</span> <span class="n">Any</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="n">response</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_result_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">mpi_disabled</span><span class="p">():</span>
|
||||
<span class="k">with</span> <span class="n">unwrap_ray_errors</span><span class="p">():</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">ray</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">request_id</span><span class="p">))</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_handle_ray_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_handle_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
|
||||
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">_aresult_step</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="s2">"The asyncio event loop was not present during initialization, so async operations are not available."</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||||
<span class="k">if</span> <span class="n">mpi_disabled</span><span class="p">():</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span><span class="o">.</span><span class="n">get_async</span><span class="o">.</span><span class="n">remote</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">request_id</span><span class="p">)</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_handle_ray_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">aqueue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
|
||||
<span class="n">global_tracer</span><span class="p">()</span><span class="o">.</span><span class="n">log_instant</span><span class="p">(</span><span class="s2">"result_step.get"</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_handle_response</span><span class="p">(</span><span class="n">response</span><span class="p">)</span>
|
||||
|
||||
@ -1426,9 +1591,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -784,9 +788,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -8763,9 +8767,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -635,9 +639,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -3498,9 +3502,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -642,9 +646,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -891,9 +895,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1358,9 +1362,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1206,9 +1210,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1232,9 +1236,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -996,9 +1000,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -651,9 +655,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -935,9 +939,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -511,6 +515,7 @@
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tqdm</span><span class="w"> </span><span class="kn">import</span> <span class="n">tqdm</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">PreTrainedTokenizerBase</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm._utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">mpi_disabled</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm.inputs.data</span><span class="w"> </span><span class="kn">import</span> <span class="n">TextPrompt</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm.inputs.multimodal</span><span class="w"> </span><span class="kn">import</span> <span class="n">MultimodalInput</span><span class="p">,</span> <span class="n">MultimodalParams</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm.inputs.registry</span><span class="w"> </span><span class="kn">import</span> <span class="n">DefaultInputProcessor</span>
|
||||
@ -628,6 +633,7 @@
|
||||
<span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Any</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_executor_cls</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"executor_cls"</span><span class="p">,</span> <span class="n">GenerationExecutor</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_orchestrator_type</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"orchestrator_type"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_llm_id</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">log_level</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">level</span>
|
||||
@ -638,6 +644,12 @@
|
||||
<span class="k">if</span> <span class="n">backend</span> <span class="o">==</span> <span class="s2">"pytorch"</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Using LLM with PyTorch backend"</span><span class="p">)</span>
|
||||
<span class="n">llm_args_cls</span> <span class="o">=</span> <span class="n">TorchLlmArgs</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_orchestrator_type</span> <span class="o">==</span> <span class="s2">"ray"</span> <span class="ow">or</span> <span class="n">mpi_disabled</span><span class="p">():</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_orchestrator_type</span> <span class="o">=</span> <span class="s2">"ray"</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"TLLM_DISABLE_MPI"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"1"</span>
|
||||
<span class="c1"># Propagate to args construction</span>
|
||||
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"orchestrator_type"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"ray"</span>
|
||||
|
||||
<span class="k">elif</span> <span class="n">backend</span> <span class="o">==</span> <span class="s1">'_autodeploy'</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Using LLM with AutoDeploy backend"</span><span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._torch.auto_deploy.llm_args</span><span class="w"> </span><span class="kn">import</span> \
|
||||
@ -758,6 +770,7 @@
|
||||
<span class="n">DisaggregatedParams</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">DisaggregatedParams</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">scheduling_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">SchedulingParams</span><span class="p">,</span>
|
||||
<span class="n">List</span><span class="p">[</span><span class="n">SchedulingParams</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">cache_salt</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="n">Union</span><span class="p">[</span><span class="n">RequestOutput</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">RequestOutput</span><span class="p">]]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Generate output for the given prompts in the synchronous mode.</span>
|
||||
<span class="sd"> Synchronous generation accepts either single prompt or batched prompts.</span>
|
||||
@ -778,6 +791,7 @@
|
||||
<span class="sd"> Disaggregated parameters. Defaults to None.</span>
|
||||
<span class="sd"> scheduling_params (tensorrt_llm.scheduling_params.SchedulingParams, List[tensorrt_llm.scheduling_params.SchedulingParams], optional):</span>
|
||||
<span class="sd"> Scheduling parameters. Defaults to None.</span>
|
||||
<span class="sd"> cache_salt (str, Sequence[str], optional): If specified, KV cache will be salted with the provided string to limit the kv cache reuse to the requests with the same string. Defaults to None.</span>
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> Union[tensorrt_llm.llmapi.RequestOutput, List[tensorrt_llm.llmapi.RequestOutput]]: The output data of the completion request to the LLM.</span>
|
||||
<span class="sd"> """</span>
|
||||
@ -808,7 +822,9 @@
|
||||
<span class="n">i</span><span class="p">),</span>
|
||||
<span class="n">disaggregated_params</span><span class="o">=</span><span class="n">_item_at</span><span class="p">(</span><span class="n">disaggregated_params</span><span class="p">,</span> <span class="n">i</span><span class="p">),</span>
|
||||
<span class="n">scheduling_params</span><span class="o">=</span><span class="n">_item_at</span><span class="p">(</span><span class="n">scheduling_params</span><span class="p">,</span> <span class="n">i</span><span class="p">),</span>
|
||||
<span class="n">streaming</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">cache_salt</span><span class="o">=</span><span class="n">_item_at</span><span class="p">(</span><span class="n">cache_salt</span><span class="p">,</span> <span class="n">i</span><span class="p">),</span>
|
||||
<span class="n">streaming</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">futures</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">future</span><span class="p">)</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">future</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="n">futures</span><span class="p">,</span>
|
||||
@ -1102,10 +1118,6 @@
|
||||
<span class="n">is_gen_only</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">backend</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"pytorch"</span><span class="p">,</span> <span class="s2">"_autodeploy"</span><span class="p">]:</span>
|
||||
<span class="k">if</span> <span class="n">sampling_params</span><span class="o">.</span><span class="n">logprobs</span> <span class="ow">and</span> <span class="n">sampling_params</span><span class="o">.</span><span class="n">logprobs</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"PyTorch backend currently only supports `logprobs=1`. Received `logprobs=</span><span class="si">{</span><span class="n">sampling_params</span><span class="o">.</span><span class="n">logprobs</span><span class="si">}</span><span class="s2">` (Top</span><span class="si">{</span><span class="n">sampling_params</span><span class="o">.</span><span class="n">logprobs</span><span class="si">}</span><span class="s2"> logprobs). Please set `logprobs=1` in `sampling_params` instead."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="c1"># Check prompt length and query length against max_num_tokens to filter illegal requests.</span>
|
||||
<span class="c1"># Skip check for gen-only requests</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">backend</span> <span class="o">==</span> <span class="s2">"pytorch"</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">enable_chunked_prefill</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_gen_only</span><span class="p">:</span>
|
||||
@ -1450,8 +1462,7 @@
|
||||
<span class="n">num_postprocess_workers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">num_postprocess_workers</span><span class="p">,</span>
|
||||
<span class="n">postprocess_tokenizer_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">postprocess_tokenizer_dir</span><span class="p">,</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">is_llm_executor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">lora_config</span><span class="o">=</span><span class="n">lora_config</span><span class="p">)</span>
|
||||
<span class="n">is_llm_executor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="nd">@append_docstring</span><span class="p">(</span><span class="n">TORCH_LLM_DOCSTRING</span><span class="p">)</span>
|
||||
@ -1492,6 +1503,34 @@
|
||||
<span class="n">backend</span><span class="o">=</span><span class="n">backend</span><span class="p">,</span>
|
||||
<span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
<span class="nd">@set_api_status</span><span class="p">(</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_collective_rpc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">args</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(),</span>
|
||||
<span class="n">kwargs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">non_block</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">unique_reply_rank</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">Any</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Execute an RPC call on all GPU workers. Currently, this is only supported for RayExecutor.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> method (str): The name of the worker method to execute.</span>
|
||||
<span class="sd"> args (tuple[Any, ...]): Positional arguments to pass to the worker method. Defaults to ().</span>
|
||||
<span class="sd"> kwargs (dict, optional): Keyword arguments to pass to the worker method. Defaults to None.</span>
|
||||
<span class="sd"> non_block (bool): Whether to block until all workers have completed the RPC call. Defaults to False.</span>
|
||||
<span class="sd"> unique_reply_rank (int, optional): The rank of the worker that will be used to send the reply. Defaults to None.</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> list[Any]: A list of results from each worker.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="p">,</span> <span class="s1">'collective_rpc'</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="o">.</span><span class="n">collective_rpc</span><span class="p">(</span><span class="n">method</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span>
|
||||
<span class="n">non_block</span><span class="p">,</span> <span class="n">unique_reply_rank</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"Executor type </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="p">)</span><span class="si">}</span><span class="s2"> does not support collective RPC."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_build_model</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">_build_model</span><span class="p">()</span>
|
||||
<span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">_engine_dir</span> <span class="ow">is</span> <span class="kc">None</span>
|
||||
@ -1525,9 +1564,6 @@
|
||||
<span class="n">postprocess_tokenizer_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">postprocess_tokenizer_dir</span><span class="p">,</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">is_llm_executor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">lora_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">lora_config</span><span class="p">,</span>
|
||||
<span class="c1"># Autodeploy does not support kv_connector_config</span>
|
||||
<span class="n">kv_connector_config</span><span class="o">=</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">,</span> <span class="s2">"kv_connector_config"</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
|
||||
<span class="n">hf_model_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">tokenizer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="p">,</span>
|
||||
<span class="n">llm_args</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="p">)</span>
|
||||
@ -1697,9 +1733,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -670,6 +674,72 @@
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">BaseSparseAttentionConfig</span><span class="p">(</span><span class="n">StrictBaseModel</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Configuration for sparse attention.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">algorithm</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"rocket"</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"rocket"</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The algorithm for sparse attention."</span><span class="p">)</span>
|
||||
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="c1"># dispatch to the correct sparse attention config</span>
|
||||
<span class="n">config_classes</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"rocket"</span><span class="p">:</span> <span class="n">RocketSparseAttentionConfig</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="n">algorithm</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"algorithm"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">algorithm</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sparse attention algorithm is required"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">config_class</span> <span class="o">=</span> <span class="n">config_classes</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">algorithm</span><span class="o">.</span><span class="n">lower</span><span class="p">())</span>
|
||||
<span class="k">if</span> <span class="n">config_class</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Invalid algorithm: </span><span class="si">{</span><span class="n">algorithm</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">config_class</span><span class="p">(</span><span class="o">**</span><span class="n">data</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_check_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">pass</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">supports_backend</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Override if the speculation algorithm does not support</span>
|
||||
<span class="sd"> a subset of the possible backends.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="RocketSparseAttentionConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.RocketSparseAttentionConfig">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">RocketSparseAttentionConfig</span><span class="p">(</span><span class="n">BaseSparseAttentionConfig</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Configuration for rocket sparse attention.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">window_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The window size for snap KV."</span><span class="p">)</span>
|
||||
<span class="n">kernel_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The kernel size for snap KV."</span><span class="p">)</span>
|
||||
<span class="n">topr</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">76</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Top-r"</span><span class="p">)</span>
|
||||
<span class="n">topk</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Top-k"</span><span class="p">)</span>
|
||||
<span class="n">prompt_budget</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">1266</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Prompt budget"</span><span class="p">)</span>
|
||||
<span class="n">page_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Page size"</span><span class="p">)</span>
|
||||
|
||||
<div class="viewcode-block" id="RocketSparseAttentionConfig.from_dict">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.RocketSparseAttentionConfig.from_dict">[docs]</a>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">**</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="RocketSparseAttentionConfig.supports_backend">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.RocketSparseAttentionConfig.supports_backend">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">supports_backend</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">backend</span> <span class="o">==</span> <span class="s2">"pytorch"</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="MoeConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.MoeConfig">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">MoeConfig</span><span class="p">(</span><span class="n">StrictBaseModel</span><span class="p">):</span>
|
||||
@ -890,7 +960,39 @@
|
||||
<span class="n">max_concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">load_format</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="c1"># PyTorch only.</span>
|
||||
<span class="c1"># Rolling average window size (N) for acceptance length across completed requests.</span>
|
||||
<span class="c1"># If not set or set to 0, the feature is disabled.</span>
|
||||
<span class="n">acceptance_window</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="c1"># PyTorch only.</span>
|
||||
<span class="c1"># Threshold for average acceptance length; speculation will be disabled</span>
|
||||
<span class="c1"># permanently once the rolling average over the last N completed requests</span>
|
||||
<span class="c1"># (N = acceptance_window) drops below this value.</span>
|
||||
<span class="n">acceptance_length_threshold</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># Validate acceptance controls at field level so they run on model creation</span>
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s1">'acceptance_window'</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_validate_acceptance_window</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]):</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">v</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"acceptance_window must be >= 0 (0 disables), got </span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">v</span>
|
||||
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s1">'acceptance_length_threshold'</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_validate_acceptance_length_threshold</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]):</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">v</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"acceptance_length_threshold must be >= 0, got </span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">v</span>
|
||||
|
||||
<span class="c1"># If set, drafting is allowed to use chain drafter.</span>
|
||||
<span class="n">_allow_chain_drafter</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="c1"># If set, drafting uses greedy sampling, irrespective of sampling parameters.</span>
|
||||
<span class="n">_allow_greedy_draft_tokens</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
@ -905,6 +1007,7 @@
|
||||
<span class="s2">"Lookahead"</span><span class="p">:</span> <span class="n">LookaheadDecodingConfig</span><span class="p">,</span>
|
||||
<span class="s2">"NGram"</span><span class="p">:</span> <span class="n">NGramDecodingConfig</span><span class="p">,</span>
|
||||
<span class="s2">"DraftTarget"</span><span class="p">:</span> <span class="n">DraftTargetDecodingConfig</span><span class="p">,</span>
|
||||
<span class="s2">"SaveState"</span><span class="p">:</span> <span class="n">SaveHiddenStatesDecodingConfig</span><span class="p">,</span>
|
||||
<span class="s2">"UserProvided"</span><span class="p">:</span> <span class="n">UserProvidedDecodingConfig</span><span class="p">,</span>
|
||||
<span class="s2">"AUTO"</span><span class="p">:</span> <span class="n">AutoDecodingConfig</span><span class="p">,</span>
|
||||
<span class="p">}</span>
|
||||
@ -1111,6 +1214,64 @@
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SaveHiddenStatesDecodingConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.SaveHiddenStatesDecodingConfig">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">SaveHiddenStatesDecodingConfig</span><span class="p">(</span><span class="n">DecodingBaseConfig</span><span class="p">):</span>
|
||||
<span class="n">output_directory</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">write_interval</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">20</span>
|
||||
<span class="n">file_prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"data"</span>
|
||||
<span class="n">eagle3_layers_to_capture</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Set</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="n">max_total_draft_tokens</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">eagle_choices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<div class="viewcode-block" id="SaveHiddenStatesDecodingConfig.model_post_init">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.SaveHiddenStatesDecodingConfig.model_post_init">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">model_post_init</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">__context</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_last_hidden_in_save</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_last_hidden_in_save</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">elif</span> <span class="o">-</span><span class="mi">1</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_last_hidden_in_save</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SaveHiddenStatesDecodingConfig.from_dict">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.SaveHiddenStatesDecodingConfig.from_dict">[docs]</a>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">**</span><span class="n">data</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="n">decoding_type</span><span class="p">:</span> <span class="n">ClassVar</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"SaveState"</span>
|
||||
|
||||
<div class="viewcode-block" id="SaveHiddenStatesDecodingConfig.validate">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.SaveHiddenStatesDecodingConfig.validate">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_directory</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="s2">"Save directory and layers to capture must be provided"</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="nd">@functools</span><span class="o">.</span><span class="n">cached_property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">spec_dec_mode</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm._torch.speculative.interface</span><span class="w"> </span><span class="kn">import</span> \
|
||||
<span class="n">SpeculativeDecodingMode</span> <span class="k">as</span> <span class="n">TorchSpeculativeDecodingMode</span>
|
||||
<span class="k">return</span> <span class="n">TorchSpeculativeDecodingMode</span><span class="o">.</span><span class="n">SAVE_HIDDEN_STATES</span>
|
||||
|
||||
<span class="nd">@functools</span><span class="o">.</span><span class="n">cached_property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">num_capture_layers</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Returns the number of layers to capture of the target model.</span>
|
||||
<span class="sd"> If eagle3_layers_to_capture is not None, return the length of the set.</span>
|
||||
<span class="sd"> Otherwise, assume Eagle3 base set and return 3 + 1 (for post norm last hidden state).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="mi">4</span>
|
||||
<span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">eagle3_layers_to_capture</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
|
||||
<div class="viewcode-block" id="UserProvidedDecodingConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.UserProvidedDecodingConfig">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">UserProvidedDecodingConfig</span><span class="p">(</span><span class="n">DecodingBaseConfig</span><span class="p">):</span>
|
||||
@ -1674,9 +1835,14 @@
|
||||
<span class="n">MTPDecodingConfig</span><span class="p">,</span>
|
||||
<span class="n">NGramDecodingConfig</span><span class="p">,</span>
|
||||
<span class="n">UserProvidedDecodingConfig</span><span class="p">,</span>
|
||||
<span class="n">SaveHiddenStatesDecodingConfig</span><span class="p">,</span>
|
||||
<span class="n">AutoDecodingConfig</span><span class="p">,</span>
|
||||
<span class="p">]]</span>
|
||||
|
||||
<span class="n">SparseAttentionConfig</span><span class="p">:</span> <span class="n">TypeAlias</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span>
|
||||
<span class="n">RocketSparseAttentionConfig</span><span class="p">,</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="KvCacheConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.KvCacheConfig">[docs]</a>
|
||||
@ -2069,6 +2235,12 @@
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Cache transceiver config."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Sparse attention config</span>
|
||||
<span class="n">sparse_attention_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">SparseAttentionConfig</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Sparse attention config."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Speculative decoding parameters</span>
|
||||
<span class="n">speculative_config</span><span class="p">:</span> <span class="n">SpeculativeConfig</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Speculative decoding config."</span><span class="p">)</span>
|
||||
@ -2087,7 +2259,7 @@
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"The maximum beam width."</span><span class="p">)</span>
|
||||
|
||||
<span class="n">max_num_tokens</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The maximum number of tokens."</span><span class="p">)</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="mi">8192</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The maximum number of tokens."</span><span class="p">)</span>
|
||||
|
||||
<span class="n">gather_generation_logits</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
@ -2141,6 +2313,13 @@
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Return perf metrics."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">orchestrator_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"rpc"</span><span class="p">,</span> <span class="s2">"ray"</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The orchestrator type to use. Defaults to None, which uses MPI."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">_parallel_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">object</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">_model_format</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_ModelFormatKind</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">_speculative_model</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
@ -2379,13 +2558,15 @@
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_batch_size</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"max_batch_size [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span><span class="si">}</span><span class="s2">] is greater than build_config.max_batch_size [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_batch_size</span><span class="si">}</span><span class="s2">] in build_config"</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_batch_size</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"max_batch_size [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span><span class="si">}</span><span class="s2">] is overridden by build_config.max_batch_size [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_batch_size</span><span class="si">}</span><span class="s2">] in build_config"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_num_tokens</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_num_tokens</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_num_tokens</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"max_num_tokens [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_num_tokens</span><span class="si">}</span><span class="s2">] is greater than build_config.max_num_tokens [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_num_tokens</span><span class="si">}</span><span class="s2">] in build_config"</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">max_num_tokens</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_num_tokens</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"max_num_tokens [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_num_tokens</span><span class="si">}</span><span class="s2">] is overridden by build_config.max_num_tokens [</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_num_tokens</span><span class="si">}</span><span class="s2">] in build_config"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_len</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_len</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_seq_len</span><span class="p">:</span>
|
||||
@ -2508,6 +2689,20 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">speculative_decoding_mode</span> <span class="o">=</span> <span class="n">SpeculativeDecodingMode</span><span class="o">.</span><span class="n">AUTO</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_draft_len</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">speculative_config</span><span class="o">.</span><span class="n">max_draft_len</span>
|
||||
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">speculative_config</span><span class="p">,</span>
|
||||
<span class="n">SaveHiddenStatesDecodingConfig</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">backend</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'pytorch'</span><span class="p">]</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="s2">"SaveHiddenStatesDecodingConfig is active, setting max_batch_size to 1, disabling overlap scheduler, and setting cuda_graph_config to None"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_batch_size</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">max_batch_size</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">disable_overlap_scheduler</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">cuda_graph_config</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">speculative_decoding_mode</span> <span class="o">=</span> <span class="n">SpeculativeDecodingMode</span><span class="o">.</span><span class="n">SAVE_HIDDEN_STATES</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">max_draft_len</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">speculative_config</span><span class="o">.</span><span class="n">max_draft_len</span> <span class="o">=</span> <span class="mi">1</span>
|
||||
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"Unrecognized speculative config type </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">speculative_config</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
@ -3592,9 +3787,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -759,9 +763,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1137,9 +1141,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -688,7 +692,7 @@
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">transformers</span><span class="o">.</span><span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_or_dir</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">)</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">)</span>
|
||||
<span class="n">hf_config_or_dir</span> <span class="o">=</span> <span class="n">hf_model_or_dir</span>
|
||||
|
||||
<span class="n">config</span> <span class="o">=</span> <span class="n">BaichuanConfig</span><span class="o">.</span><span class="n">from_hugging_face</span><span class="p">(</span><span class="n">hf_config_or_dir</span><span class="p">,</span>
|
||||
@ -869,9 +873,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1173,9 +1177,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -781,9 +785,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -798,9 +802,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -808,7 +812,7 @@
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">AutoModel</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_or_dir</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s1">'auto'</span> <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">chatglm_version</span> <span class="o">!=</span> <span class="s1">'glm'</span> <span class="k">else</span> <span class="nb">getattr</span><span class="p">(</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="s1">'auto'</span> <span class="k">if</span> <span class="n">config</span><span class="o">.</span><span class="n">chatglm_version</span> <span class="o">!=</span> <span class="s1">'glm'</span> <span class="k">else</span> <span class="nb">getattr</span><span class="p">(</span>
|
||||
<span class="n">torch</span><span class="p">,</span> <span class="n">config</span><span class="o">.</span><span class="n">dtype</span><span class="p">),</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="n">device_map</span><span class="p">)</span>
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">load_weights_from_hf_model</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
|
||||
@ -997,9 +1001,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -826,9 +830,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -657,9 +661,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -910,9 +914,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -808,9 +812,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -672,9 +676,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -798,9 +802,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -892,9 +896,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -974,9 +978,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1010,9 +1014,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1946,9 +1950,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -2853,9 +2857,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -733,9 +737,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -776,7 +780,7 @@
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">load_weights_from_hf_by_shard</span><span class="p">(</span><span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">transformers</span><span class="o">.</span><span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">torch_dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">)</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">)</span>
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">load_weights_from_hf_model</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
|
||||
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
|
||||
@ -895,9 +899,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -823,9 +827,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -802,7 +806,7 @@
|
||||
<span class="n">hf_gemma</span> <span class="o">=</span> <span class="n">transformers</span><span class="o">.</span><span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s2">"cpu"</span> <span class="k">if</span> <span class="n">load_model_on_cpu</span> <span class="k">else</span> <span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">load_gemma_weights_from_hf_model</span><span class="p">(</span><span class="n">hf_gemma</span><span class="p">,</span> <span class="n">trt_llm_config</span><span class="p">)</span>
|
||||
<span class="k">del</span> <span class="n">hf_gemma</span>
|
||||
@ -1018,9 +1022,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -942,9 +946,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1045,9 +1049,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -671,9 +675,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -702,9 +706,7 @@
|
||||
<span class="n">trust_remote_code</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'trust_remote_code'</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">transformers</span><span class="o">.</span><span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">'auto'</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">load_weights_from_hf_model</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">config</span><span class="p">)</span>
|
||||
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">GPTJForCausalLM</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
|
||||
@ -823,9 +825,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -763,9 +767,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -897,9 +901,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1245,9 +1249,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -966,7 +970,7 @@
|
||||
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">hf_model_dir</span><span class="p">):</span>
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">torch_dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">transformers</span><span class="o">.</span><span class="n">PreTrainedModel</span><span class="p">)</span>
|
||||
<span class="n">weights</span> <span class="o">=</span> <span class="n">convert_hf_mamba</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">dtype</span><span class="p">)</span>
|
||||
@ -1090,9 +1094,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -730,9 +734,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -726,7 +730,7 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">transformers</span><span class="o">.</span><span class="n">PreTrainedModel</span><span class="p">)</span>
|
||||
@ -880,9 +884,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -2191,9 +2195,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1257,9 +1261,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -598,6 +602,7 @@
|
||||
<span class="n">EAGLE</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
|
||||
<span class="n">NGRAM</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
|
||||
<span class="n">USER_PROVIDED</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
|
||||
<span class="n">SAVE_HIDDEN_STATES</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
|
||||
<span class="n">AUTO</span> <span class="o">=</span> <span class="n">auto</span><span class="p">()</span>
|
||||
|
||||
<div class="viewcode-block" id="SpeculativeDecodingMode.from_arguments">
|
||||
@ -622,6 +627,8 @@
|
||||
<span class="k">return</span> <span class="n">SpeculativeDecodingMode</span><span class="o">.</span><span class="n">USER_PROVIDED</span>
|
||||
<span class="k">elif</span> <span class="n">args</span><span class="o">.</span><span class="n">speculative_decoding_mode</span> <span class="o">==</span> <span class="s2">"auto"</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">SpeculativeDecodingMode</span><span class="o">.</span><span class="n">AUTO</span>
|
||||
<span class="k">elif</span> <span class="n">args</span><span class="o">.</span><span class="n">speculative_decoding_mode</span> <span class="o">==</span> <span class="s2">"save_hidden_states"</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">SpeculativeDecodingMode</span><span class="o">.</span><span class="n">SAVE_HIDDEN_STATES</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">assert</span> <span class="kc">False</span><span class="p">,</span> <span class="s2">"Unknown speculative_decoding_mode "</span> <span class="o">+</span> <span class="n">args</span><span class="o">.</span><span class="n">speculative_decoding_mode</span></div>
|
||||
</div>
|
||||
@ -2665,9 +2672,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -795,9 +799,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -729,9 +733,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -797,9 +801,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -800,9 +804,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -714,9 +718,7 @@
|
||||
<span class="n">trust_remote_code</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'trust_remote_code'</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">transformers</span><span class="o">.</span><span class="n">PreTrainedModel</span><span class="p">)</span>
|
||||
|
||||
@ -844,9 +846,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -810,9 +814,7 @@
|
||||
<span class="n">trust_remote_code</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">'trust_remote_code'</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="n">hf_model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
<span class="n">hf_model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span> <span class="n">trust_remote_code</span><span class="o">=</span><span class="n">trust_remote_code</span><span class="p">)</span>
|
||||
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hf_model</span><span class="p">,</span> <span class="n">transformers</span><span class="o">.</span><span class="n">PreTrainedModel</span><span class="p">)</span>
|
||||
|
||||
@ -940,9 +942,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1243,9 +1247,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -930,9 +934,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -515,13 +519,15 @@
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">platform</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">collections</span><span class="w"> </span><span class="kn">import</span> <span class="n">OrderedDict</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">asdict</span><span class="p">,</span> <span class="n">dataclass</span><span class="p">,</span> <span class="n">field</span><span class="p">,</span> <span class="n">fields</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">enum</span><span class="w"> </span><span class="kn">import</span> <span class="n">IntEnum</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">pathlib</span><span class="w"> </span><span class="kn">import</span> <span class="n">Path</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">textwrap</span><span class="w"> </span><span class="kn">import</span> <span class="n">dedent</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Tuple</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">Any</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Literal</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">get_args</span><span class="p">,</span>
|
||||
<span class="n">get_origin</span><span class="p">)</span>
|
||||
|
||||
<span class="kn">import</span><span class="w"> </span><span class="nn">tensorrt</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">trt</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">pydantic</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">BaseModel</span><span class="p">,</span> <span class="n">ConfigDict</span><span class="p">,</span> <span class="n">Field</span><span class="p">,</span> <span class="n">PrivateAttr</span><span class="p">,</span> <span class="n">ValidationInfo</span><span class="p">,</span>
|
||||
<span class="n">field_validator</span><span class="p">)</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._ipc_utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">IpcMemory</span><span class="p">,</span> <span class="n">can_access_peer</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">get_sm_version</span>
|
||||
@ -577,68 +583,13 @@
|
||||
<span class="n">enabled_with_fp32_acc</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
|
||||
|
||||
<span class="n">DEFAULT_PLUGIN_DTYPE_OPTIONS</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"float32"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span> <span class="s2">"int32"</span><span class="p">,</span> <span class="kc">None</span>
|
||||
<span class="p">]</span>
|
||||
<span class="n">PLUGIN_DTYPE_OPTIONS_MAP</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="s2">"gemm_swiglu_plugin"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span>
|
||||
<span class="s2">"gemm_plugin"</span><span class="p">:</span>
|
||||
<span class="p">[</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"float32"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span> <span class="s2">"int32"</span><span class="p">,</span> <span class="s2">"fp8"</span><span class="p">,</span> <span class="s2">"nvfp4"</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span>
|
||||
<span class="s2">"low_latency_gemm_plugin"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span>
|
||||
<span class="s2">"low_latency_gemm_swiglu_plugin"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span>
|
||||
<span class="s2">"gemm_allreduce_plugin"</span><span class="p">:</span> <span class="p">[</span><span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_make_plugin_property</span><span class="p">(</span><span class="n">field_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">field_type</span><span class="p">:</span> <span class="nb">type</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">bind</span><span class="p">(</span><span class="n">field_name</span><span class="p">):</span>
|
||||
<span class="n">storage_name</span> <span class="o">=</span> <span class="sa">f</span><span class="s1">'_</span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s1">'</span>
|
||||
|
||||
<span class="nd">@property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">prop</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="n">field_value</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">storage_name</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="o">!=</span> <span class="s1">'dtype'</span> <span class="ow">and</span> <span class="n">field_value</span> <span class="o">==</span> <span class="s1">'auto'</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dtype</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">field_value</span>
|
||||
|
||||
<span class="nd">@prop</span><span class="o">.</span><span class="n">setter</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">prop</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">field_type</span> <span class="ow">is</span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">bool</span><span class="p">),</span> \
|
||||
<span class="sa">f</span><span class="s2">"Plugin </span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2"> expects </span><span class="si">{</span><span class="n">field_type</span><span class="si">}</span><span class="s2">, got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">elif</span> <span class="n">field_type</span> <span class="ow">in</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="n">DEFAULT_PLUGIN_DTYPE_OPTIONS</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="n">PLUGIN_DTYPE_OPTIONS_MAP</span><span class="p">:</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="n">PLUGIN_DTYPE_OPTIONS_MAP</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span>
|
||||
<span class="k">assert</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">plugin_dtype_options</span><span class="p">,</span> \
|
||||
<span class="sa">f</span><span class="s2">"Plugin </span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2"> expects values in </span><span class="si">{</span><span class="n">plugin_dtype_options</span><span class="si">}</span><span class="s2">, got </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s1">'dtype'</span><span class="p">:</span>
|
||||
<span class="k">assert</span> <span class="n">value</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">'auto'</span><span class="p">,</span> <span class="kc">None</span><span class="p">],</span> \
|
||||
<span class="s2">"Plugin dtype cannot be auto or None"</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">storage_name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Set </span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">value</span><span class="si">}</span><span class="s2">."</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">prop</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">bind</span><span class="p">(</span><span class="n">field_name</span><span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">PluginConfigMeta</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">):</span>
|
||||
<span class="k">for</span> <span class="n">storage_name</span><span class="p">,</span> <span class="n">field_type</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">[</span><span class="s1">'__annotations__'</span><span class="p">]</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">assert</span> <span class="n">storage_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span>
|
||||
<span class="n">field_name</span> <span class="o">=</span> <span class="n">storage_name</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span>
|
||||
<span class="n">attrs</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_make_plugin_property</span><span class="p">(</span><span class="n">field_name</span><span class="p">,</span> <span class="n">field_type</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">attrs</span><span class="p">)</span>
|
||||
<span class="n">DefaultPluginDtype</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"float32"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span> <span class="s2">"int32"</span><span class="p">,</span>
|
||||
<span class="kc">None</span><span class="p">]</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig">[docs]</a>
|
||||
<span class="nd">@dataclass</span><span class="p">(</span><span class="n">slots</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">PluginConfig</span><span class="p">(</span><span class="n">metaclass</span><span class="o">=</span><span class="n">PluginConfigMeta</span><span class="p">):</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">PluginConfig</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""The config that manages plugin-related options.</span>
|
||||
|
||||
<span class="sd"> There are two option categories:</span>
|
||||
@ -649,356 +600,291 @@
|
||||
<span class="sd"> * Other features. These options can be assigned with boolean:</span>
|
||||
<span class="sd"> * True, which means the plugin is enabled;</span>
|
||||
<span class="sd"> * False, which means the plugin is disabled.</span>
|
||||
|
||||
<span class="sd"> Note: All the fields should use a prefix "_"; PluginConfigMeta will wrap each field as a property.</span>
|
||||
<span class="sd"> This ensures the fields can only be assigned with allowed values.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">_dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="s2">"float16"</span><span class="p">,</span> <span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">model_config</span> <span class="o">=</span> <span class="n">ConfigDict</span><span class="p">(</span><span class="n">validate_assignment</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">extra</span><span class="o">=</span><span class="s2">"ignore"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="s2">"float16"</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Base dtype for the model and plugins"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plugins</span>
|
||||
<span class="n">_bert_attention_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">bert_attention_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The plugin that uses efficient kernels and enables an in-place update of the KV cache for attention layer of BERT-like encoder models."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_gpt_attention_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The plugin that uses efficient kernels and enables an in-place update of the KV cache for attention layer of BERT-like encoder models."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">gpt_attention_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The plugin that uses efficient kernels and enables an in-place update of the KV cache for attention layer of GPT-like decoder models."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The plugin that uses efficient kernels and enables an in-place update of the KV cache for attention layer of GPT-like decoder models."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span>
|
||||
<span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"float32"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span> <span class="s2">"int32"</span><span class="p">,</span> <span class="s2">"fp8"</span><span class="p">,</span> <span class="s2">"nvfp4"</span><span class="p">,</span>
|
||||
<span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The GEMM plugin that utilizes NVIDIA cuBLASLt to perform GEMM operations. "</span>
|
||||
<span class="s2">"Note: it's only affective for non-quantized gemm operations (except FP8)."</span>
|
||||
<span class="s2">"Note: For FP8, it also requires same calibration in checkpoint."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_explicitly_disable_gemm_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">_gemm_swiglu_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="s2">"Note: For FP8, it also requires same calibration in checkpoint."</span><span class="p">)</span>
|
||||
<span class="n">_explicitly_disable_gemm_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">gemm_swiglu_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The GEMM + SwiGLU fusion in Gated-MLP combines two Matmul operations and "</span>
|
||||
<span class="s2">"one SwiGLU operation into a single kernel. Currently this is only supported for FP8 precision on Hopper."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_fp8_rowwise_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The GEMM + SwiGLU fusion in Gated-MLP combines two Matmul operations and "</span>
|
||||
<span class="s2">"one SwiGLU operation into a single kernel. Currently this is only supported for FP8 precision on Hopper."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">fp8_rowwise_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The quantized GEMM for fp8, which uses per token dynamic scales for "</span>
|
||||
<span class="s2">"activation and per channel static scales for weights."</span>
|
||||
<span class="s2">"Note: It also requires same calibration in checkpoint."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_qserve_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The quantized GEMM for fp8, which uses per token dynamic scales for "</span>
|
||||
<span class="s2">"activation and per channel static scales for weights."</span>
|
||||
<span class="s2">"Note: It also requires same calibration in checkpoint."</span><span class="p">)</span>
|
||||
<span class="n">qserve_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The quantized GEMM from [QServe](https://arxiv.org/abs/2405.04532), "</span>
|
||||
<span class="s2">"which employs 4-bit quantization for weights and 8-bit quantization for activations."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_identity_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The quantized GEMM from [QServe](https://arxiv.org/abs/2405.04532), "</span>
|
||||
<span class="s2">"which employs 4-bit quantization for weights and 8-bit quantization for activations."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">identity_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The identity plugin simply copies inputs to outputs, it's used mostly for debugging purpose."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_nccl_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The identity plugin simply copies inputs to outputs, it's used mostly for debugging purpose."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">nccl_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The NCCL plugin wraps NCCL operators to support multi-GPU and even multi-nodes."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_lora_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable LoRA."</span><span class="p">})</span>
|
||||
<span class="n">_dora_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable DoRA."</span><span class="p">})</span>
|
||||
<span class="n">_weight_only_groupwise_quant_matmul_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The NCCL plugin wraps NCCL operators to support multi-GPU and even multi-nodes."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">lora_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Enable LoRA."</span><span class="p">)</span>
|
||||
<span class="n">dora_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"Enable DoRA."</span><span class="p">)</span>
|
||||
<span class="n">weight_only_groupwise_quant_matmul_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
|
||||
<span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable weight-only groupwise quantization matmul operators."</span><span class="p">)</span>
|
||||
<span class="n">weight_only_quant_matmul_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable weight-only groupwise quantization matmul operators."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_weight_only_quant_matmul_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable weight-only quantization matmul operators."</span><span class="p">})</span>
|
||||
<span class="n">_smooth_quant_plugins</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable weight-only quantization matmul operators."</span><span class="p">)</span>
|
||||
<span class="n">smooth_quant_plugins</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable a group of plugins to support smooth quantization."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_smooth_quant_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable a group of plugins to support smooth quantization."</span><span class="p">)</span>
|
||||
<span class="n">smooth_quant_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable plugin that supports smooth quantization gemm kernels."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_layernorm_quantization_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable plugin that supports smooth quantization gemm kernels."</span><span class="p">)</span>
|
||||
<span class="n">layernorm_quantization_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable plugin that supports layernorm quantization kernels."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_rmsnorm_quantization_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable plugin that supports layernorm quantization kernels."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">rmsnorm_quantization_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable plugin that supports rmsnorm quantization kernels."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_quantize_per_token_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable plugin that supports rmsnorm quantization kernels."</span><span class="p">)</span>
|
||||
<span class="n">quantize_per_token_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable plugin that supports per-token quantization."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_quantize_tensor_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable plugin that supports per-token quantization."</span><span class="p">)</span>
|
||||
<span class="n">quantize_tensor_plugin</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Enable plugin that supports per-tensor quantization."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_moe_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable plugin that supports per-tensor quantization."</span><span class="p">)</span>
|
||||
<span class="n">moe_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable some customized kernels to speed up the MoE layer of MoE models."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_mamba_conv1d_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable some customized kernels to speed up the MoE layer of MoE models."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">mamba_conv1d_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">DefaultPluginDtype</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable customized kernels to speed up conv1d operator for Mamba."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_low_latency_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable customized kernels to speed up conv1d operator for Mamba."</span><span class="p">)</span>
|
||||
<span class="n">low_latency_gemm_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The GEMM plugin that optimized specially for low latency scenarios."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_low_latency_gemm_swiglu_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The GEMM plugin that optimized specially for low latency scenarios."</span><span class="p">)</span>
|
||||
<span class="n">low_latency_gemm_swiglu_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span><span class="s2">"fp8"</span><span class="p">,</span> <span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"The GEMM + SwiGLU fusion plugin that optimized specially for low latency scenarios."</span>
|
||||
<span class="p">})</span>
|
||||
|
||||
<span class="n">_gemm_allreduce_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span><span class="s2">"help"</span><span class="p">:</span> <span class="s2">"The GEMM + AllReduce kernel fusion plugin."</span><span class="p">})</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The GEMM + SwiGLU fusion plugin that optimized specially for low latency scenarios."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">gemm_allreduce_plugin</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Literal</span><span class="p">[</span>
|
||||
<span class="s2">"float16"</span><span class="p">,</span> <span class="s2">"bfloat16"</span><span class="p">,</span>
|
||||
<span class="kc">None</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"The GEMM + AllReduce kernel fusion plugin."</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Features</span>
|
||||
<span class="n">_context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable the fused multi-head attention during the context phase, "</span>
|
||||
<span class="s2">"will trigger a kernel that performs the MHA/MQA/GQA block using a single kernel."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_bert_context_fmha_fp32_acc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable the fused multi-head attention during the context phase, "</span>
|
||||
<span class="s2">"will trigger a kernel that performs the MHA/MQA/GQA block using a single kernel."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">bert_context_fmha_fp32_acc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable the FP32 accumulator for context FMHA in the bert_attention_plugin. "</span>
|
||||
<span class="s2">"If disabled, FP16 is used, better performance but potentially worse accuracy is expected."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_paged_kv_cache</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable the FP32 accumulator for context FMHA in the bert_attention_plugin. "</span>
|
||||
<span class="s2">"If disabled, FP16 is used, better performance but potentially worse accuracy is expected."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">paged_kv_cache</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable paged KV cache, which helps manage memory for the KV cache more efficiently, "</span>
|
||||
<span class="s2">"and usually leads to an increase in the batch size and an improved efficiency."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_remove_input_padding</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable paged KV cache, which helps manage memory for the KV cache more efficiently, "</span>
|
||||
<span class="s2">"and usually leads to an increase in the batch size and an improved efficiency."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">remove_input_padding</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Pack different tokens together, which reduces both the amount of computations and memory consumption."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_norm_quant_fusion</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Pack different tokens together, which reduces both the amount of computations and memory consumption."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">norm_quant_fusion</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Fuse the LayerNorm and quantization kernels into a single kernel, "</span>
|
||||
<span class="s2">"resulting in improved end-to-end performance."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_reduce_fusion</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Fuse the LayerNorm and quantization kernels into a single kernel, "</span>
|
||||
<span class="s2">"resulting in improved end-to-end performance."</span><span class="p">)</span>
|
||||
<span class="n">reduce_fusion</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Fuse the ResidualAdd and LayerNorm kernels after AllReduce into a single kernel, "</span>
|
||||
<span class="s2">"resulting in improved end-to-end performance."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_user_buffer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Fuse the ResidualAdd and LayerNorm kernels after AllReduce into a single kernel, "</span>
|
||||
<span class="s2">"resulting in improved end-to-end performance."</span><span class="p">)</span>
|
||||
<span class="n">user_buffer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Eliminate extra copies from the local buffer to the shared buffer "</span>
|
||||
<span class="s2">"in the communication kernel, leading to improved end-to-end performance. "</span>
|
||||
<span class="s2">"This feature must be enabled with `--reduce_fusion enable` and "</span>
|
||||
<span class="s2">"is currently only supported for the FP8 LLAMA model."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_tokens_per_block</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Eliminate extra copies from the local buffer to the shared buffer "</span>
|
||||
<span class="s2">"in the communication kernel, leading to improved end-to-end performance. "</span>
|
||||
<span class="s2">"This feature must be enabled with `--reduce_fusion enable` and "</span>
|
||||
<span class="s2">"is currently only supported for the FP8 LLAMA model."</span><span class="p">)</span>
|
||||
<span class="n">tokens_per_block</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Define how many tokens are contained in each paged kv cache block."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_use_paged_context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Define how many tokens are contained in each paged kv cache block."</span><span class="p">)</span>
|
||||
<span class="n">use_paged_context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Allow advanced features like KV cache reuse and chunked context."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_use_fp8_context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Allow advanced features like KV cache reuse and chunked context."</span><span class="p">)</span>
|
||||
<span class="n">use_fp8_context_fmha</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"When FP8 quantization is activated, the attention can be further accelerated by enabling FP8 Context FMHA"</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_fuse_fp4_quant</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"When FP8 quantization is activated, the attention can be further accelerated by enabling FP8 Context FMHA"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">fuse_fp4_quant</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span> <span class="s2">"Whether to fuse FP4 quantization into attention kernel."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_multiple_profiles</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Whether to fuse FP4 quantization into attention kernel."</span><span class="p">)</span>
|
||||
<span class="n">multiple_profiles</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enables multiple TensorRT optimization profiles in the built engines, "</span>
|
||||
<span class="s2">"will benefits the performance especially when GEMM plugin is disabled, "</span>
|
||||
<span class="s2">"because more optimization profiles help TensorRT have more chances to select better kernels. "</span>
|
||||
<span class="s2">"Note: This feature increases engine build time but no other adverse effects are expected."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_paged_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enables multiple TensorRT optimization profiles in the built engines, "</span>
|
||||
<span class="s2">"will benefits the performance especially when GEMM plugin is disabled, "</span>
|
||||
<span class="s2">"because more optimization profiles help TensorRT have more chances to select better kernels. "</span>
|
||||
<span class="s2">"Note: This feature increases engine build time but no other adverse effects are expected."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">paged_state</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable paged state, which helps manage memory for the RNN state more efficiently."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_streamingllm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable paged state, which helps manage memory for the RNN state more efficiently."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">streamingllm</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable [StreamingLLM](https://arxiv.org/abs/2309.17453), which uses a window attention to perform efficient and stable LLM on long texts."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_manage_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable [StreamingLLM](https://arxiv.org/abs/2309.17453), which uses a window attention to perform efficient and stable LLM on long texts."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">manage_weights</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable TensorRT LLM managed weights to speed up engine building process."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_use_fused_mlp</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable TensorRT LLM managed weights to speed up engine building process."</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">use_fused_mlp</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable horizontal fusion in Gated-MLP that combines two Matmul "</span>
|
||||
<span class="s2">"operations into a single one followed by a separate SwiGLU kernel."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">_pp_reduce_scatter</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">field</span><span class="p">(</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Enable horizontal fusion in Gated-MLP that combines two Matmul "</span>
|
||||
<span class="s2">"operations into a single one followed by a separate SwiGLU kernel."</span><span class="p">)</span>
|
||||
<span class="n">pp_reduce_scatter</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">init</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">metadata</span><span class="o">=</span><span class="p">{</span>
|
||||
<span class="s2">"help"</span><span class="p">:</span>
|
||||
<span class="s2">"Enable a pipeline parallelism optimization with "</span>
|
||||
<span class="s2">"ReduceScatter + AllGather targeting large MoE models."</span>
|
||||
<span class="p">})</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable a pipeline parallelism optimization with "</span>
|
||||
<span class="s2">"ReduceScatter + AllGather targeting large MoE models."</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">update_from_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">config</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">config</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
|
||||
<span class="n">value_to_be_update</span> <span class="o">=</span> <span class="n">config</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">),</span>
|
||||
<span class="nb">bool</span><span class="p">)</span> <span class="ow">or</span> <span class="n">name</span> <span class="o">==</span> <span class="s1">'paged_kv_cache'</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">value_to_be_update</span> <span class="o">==</span> <span class="s2">"enable"</span><span class="p">:</span>
|
||||
<span class="n">value_to_be_update</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">elif</span> <span class="n">value_to_be_update</span> <span class="o">==</span> <span class="s2">"disable"</span><span class="p">:</span>
|
||||
<span class="n">value_to_be_update</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="k">elif</span> <span class="n">value_to_be_update</span> <span class="o">==</span> <span class="s2">"disable"</span><span class="p">:</span>
|
||||
<span class="n">value_to_be_update</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">value_to_be_update</span><span class="p">)</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Override to resolve 'auto' values to dtype field.</span>
|
||||
|
||||
<span class="sd"> When a plugin field has value 'auto', return the value of dtype instead.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># Use object.__getattribute__ to avoid infinite recursion</span>
|
||||
<span class="n">value</span> <span class="o">=</span> <span class="nb">object</span><span class="o">.</span><span class="fm">__getattribute__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">name</span> <span class="o">!=</span> <span class="s2">"dtype"</span> <span class="ow">and</span> <span class="n">value</span> <span class="o">==</span> <span class="s2">"auto"</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">dtype</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">value</span>
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.validate_dtype_not_auto">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.validate_dtype_not_auto">[docs]</a>
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s2">"dtype"</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">config</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
<span class="n">plugin_config</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">()</span>
|
||||
<span class="n">plugin_config</span><span class="o">.</span><span class="n">update_from_dict</span><span class="p">(</span><span class="n">config</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">plugin_config</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate_dtype_not_auto</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="o">==</span> <span class="s2">"auto"</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Plugin dtype cannot be 'auto'"</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">v</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.convert_enable_disable">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.convert_enable_disable">[docs]</a>
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s2">"*"</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"before"</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">convert_enable_disable</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">info</span><span class="p">:</span> <span class="n">ValidationInfo</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Allow passing enable/disable strings which map to boolean/None values."""</span>
|
||||
<span class="k">if</span> <span class="n">value</span> <span class="o">==</span> <span class="s2">"enable"</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="kc">True</span>
|
||||
<span class="k">elif</span> <span class="n">value</span> <span class="o">==</span> <span class="s2">"disable"</span><span class="p">:</span>
|
||||
<span class="n">annotation</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">model_fields</span><span class="p">[</span><span class="n">info</span><span class="o">.</span><span class="n">field_name</span><span class="p">]</span><span class="o">.</span><span class="n">annotation</span>
|
||||
<span class="k">if</span> <span class="n">annotation</span> <span class="ow">is</span> <span class="nb">bool</span> <span class="ow">or</span> <span class="p">(</span><span class="n">get_origin</span><span class="p">(</span><span class="n">annotation</span><span class="p">)</span> <span class="ow">is</span> <span class="n">Union</span>
|
||||
<span class="ow">and</span> <span class="nb">bool</span> <span class="ow">in</span> <span class="n">get_args</span><span class="p">(</span><span class="n">annotation</span><span class="p">)):</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
<span class="k">return</span> <span class="kc">None</span>
|
||||
<span class="k">return</span> <span class="n">value</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.log_field_changes">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.log_field_changes">[docs]</a>
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s2">"*"</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s2">"after"</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">log_field_changes</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="n">info</span><span class="p">:</span> <span class="n">ValidationInfo</span><span class="p">)</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Log all field changes for debugging."""</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Set </span><span class="si">{</span><span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="n">info</span><span class="o">.</span><span class="n">field_name</span><span class="si">}</span><span class="s2"> to </span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">."</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">v</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.from_arguments">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.from_arguments">[docs]</a>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_arguments</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">args</span><span class="p">:</span> <span class="n">argparse</span><span class="o">.</span><span class="n">Namespace</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Create a PluginConfig from argparse arguments."""</span>
|
||||
<span class="n">args</span> <span class="o">=</span> <span class="nb">vars</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">from_dict</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||||
<span class="n">obj</span> <span class="o">=</span> <span class="bp">cls</span><span class="p">(</span><span class="o">**</span><span class="n">args</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We want to know if the user explicitly disabled the gemm_plugin</span>
|
||||
<span class="c1"># because nvfp4 gemm uses plugin by default currently</span>
|
||||
<span class="k">if</span> <span class="s1">'gemm_plugin'</span> <span class="ow">in</span> <span class="n">args</span> <span class="ow">and</span> <span class="n">args</span><span class="p">[</span><span class="s1">'gemm_plugin'</span><span class="p">]</span> <span class="o">==</span> <span class="s1">'disable'</span><span class="p">:</span>
|
||||
<span class="n">obj</span><span class="o">.</span><span class="n">_explicitly_disable_gemm_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">obj</span>
|
||||
<span class="k">return</span> <span class="n">obj</span></div>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="n">config</span> <span class="o">=</span> <span class="n">asdict</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
|
||||
<span class="c1"># Remove prefix "_" of the storage name</span>
|
||||
<span class="n">config</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">'_'</span><span class="p">):</span> <span class="n">value</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">config</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||
<span class="k">return</span> <span class="n">config</span>
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.to_legacy_setting">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.to_legacy_setting">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">to_legacy_setting</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">'''Legacy setting means that all of the plugins and features are</span>
|
||||
<span class="w"> </span><span class="sd">"""Legacy setting means that all of the plugins and features are</span>
|
||||
<span class="sd"> disabled, this is needed for the legacy `build.py` script, which will be</span>
|
||||
<span class="sd"> migrated to the centralized building script `tensorrt_llm/commands/build.py`.</span>
|
||||
|
||||
<span class="sd"> After the migration is done, this function may or may not be deleted.</span>
|
||||
<span class="sd"> '''</span>
|
||||
<span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># Remove prefix "_" of the storage name</span>
|
||||
<span class="n">field_name</span> <span class="o">=</span> <span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s1">'dtype'</span><span class="p">:</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">for</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">field_value</span> <span class="ow">in</span> <span class="bp">self</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s2">"dtype"</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_value</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">field</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="nb">bool</span> <span class="ow">or</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s1">'paged_kv_cache'</span><span class="p">:</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">field_value</span><span class="p">,</span>
|
||||
<span class="nb">bool</span><span class="p">)</span> <span class="ow">or</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s2">"paged_kv_cache"</span><span class="p">:</span>
|
||||
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field_name</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.validate">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.validate">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="n">unsupported_plugins</span> <span class="o">=</span> <span class="p">{</span>
|
||||
<span class="c1"># bert_attention_plugin is handled within BertAttention</span>
|
||||
@ -1014,7 +900,8 @@
|
||||
<span class="n">val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">plugin</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">val</span> <span class="o">!=</span> <span class="kc">False</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">plugin</span><span class="si">}</span><span class="s2">=</span><span class="si">{</span><span class="n">val</span><span class="si">}</span><span class="s2"> is not supported on SM </span><span class="si">{</span><span class="n">sm</span><span class="si">}</span><span class="s2">."</span><span class="p">)</span>
|
||||
<span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">plugin</span><span class="si">}</span><span class="s2">=</span><span class="si">{</span><span class="n">val</span><span class="si">}</span><span class="s2"> is not supported on SM </span><span class="si">{</span><span class="n">sm</span><span class="si">}</span><span class="s2">."</span><span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="nd">@property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">context_fmha_type</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
@ -1025,8 +912,11 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">ContextFMHAType</span><span class="o">.</span><span class="n">disabled</span>
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.is_context_fmha_enabled">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.is_context_fmha_enabled">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">is_context_fmha_enabled</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">context_fmha_type</span> <span class="o">!=</span> <span class="n">ContextFMHAType</span><span class="o">.</span><span class="n">disabled</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">context_fmha_type</span> <span class="o">!=</span> <span class="n">ContextFMHAType</span><span class="o">.</span><span class="n">disabled</span></div>
|
||||
|
||||
|
||||
<span class="nd">@context_fmha_type</span><span class="o">.</span><span class="n">setter</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">context_fmha_type</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
|
||||
@ -1040,50 +930,74 @@
|
||||
<span class="k">elif</span> <span class="n">value</span> <span class="o">==</span> <span class="n">ContextFMHAType</span><span class="o">.</span><span class="n">enabled_with_fp32_acc</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">bert_context_fmha_fp32_acc</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_smooth_quant_plugins">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_smooth_quant_plugins">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_smooth_quant_plugins</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"auto"</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">smooth_quant_gemm_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">rmsnorm_quantization_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">layernorm_quantization_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">quantize_per_token_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">quantize_tensor_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_qserve_plugins">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_qserve_plugins">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_qserve_plugins</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"auto"</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">qserve_gemm_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">rmsnorm_quantization_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">quantize_per_token_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_fp8_rowwise_quant_plugins">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_fp8_rowwise_quant_plugins">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_fp8_rowwise_quant_plugins</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"auto"</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">fp8_rowwise_gemm_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">rmsnorm_quantization_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">layernorm_quantization_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">quantize_per_token_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">quantize_tensor_plugin</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_context_fmha">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_context_fmha">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_context_fmha</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context_fmha_type</span><span class="o">=</span><span class="n">ContextFMHAType</span><span class="o">.</span><span class="n">enabled</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="n">context_fmha_type</span><span class="p">)</span> <span class="o">==</span> <span class="n">ContextFMHAType</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">context_fmha_type</span> <span class="o">=</span> <span class="n">context_fmha_type</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.enable_paged_kv_cache">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.enable_paged_kv_cache">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">enable_paged_kv_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tokens_per_block</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">32</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">paged_kv_cache</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">tokens_per_block</span> <span class="o">=</span> <span class="n">tokens_per_block</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_nccl_plugin">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_nccl_plugin">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_nccl_plugin</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"auto"</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">nccl_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="n">init_all_reduce_helper</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_lora_plugin">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_lora_plugin">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_lora_plugin</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dtype</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">lora_plugin</span> <span class="o">=</span> <span class="n">dtype</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="PluginConfig.set_dora_plugin">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.plugin.html#tensorrt_llm.plugin.PluginConfig.set_dora_plugin">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">set_dora_plugin</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">enable</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">dora_plugin</span> <span class="o">=</span> <span class="n">enable</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@ -1125,41 +1039,47 @@
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">add_plugin_argument</span><span class="p">(</span><span class="n">parser</span><span class="p">:</span> <span class="n">argparse</span><span class="o">.</span><span class="n">ArgumentParser</span><span class="p">):</span>
|
||||
<span class="n">plugin_config</span> <span class="o">=</span> <span class="n">PluginConfig</span><span class="p">()</span>
|
||||
<span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">fields</span><span class="p">(</span><span class="n">plugin_config</span><span class="p">):</span>
|
||||
<span class="c1"># Remove prefix "_" of the storage name</span>
|
||||
<span class="n">field_name</span> <span class="o">=</span> <span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">field_name</span><span class="p">,</span> <span class="n">field_info</span> <span class="ow">in</span> <span class="n">PluginConfig</span><span class="o">.</span><span class="n">model_fields</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">cli_plugin_args</span><span class="p">:</span>
|
||||
<span class="k">continue</span>
|
||||
<span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">metadata</span> <span class="ow">and</span> <span class="s2">"help"</span> <span class="ow">in</span> <span class="n">field</span><span class="o">.</span><span class="n">metadata</span><span class="p">:</span>
|
||||
<span class="n">help_message</span> <span class="o">=</span> <span class="n">field</span><span class="o">.</span><span class="n">metadata</span><span class="p">[</span><span class="s2">"help"</span><span class="p">]</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">help_message</span> <span class="o">=</span> <span class="n">field_info</span><span class="o">.</span><span class="n">description</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="n">help_message</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">AttributeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Please add help message for </span><span class="si">{</span><span class="n">field_name</span><span class="si">}</span><span class="s2">."</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">type</span> <span class="ow">in</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="n">DEFAULT_PLUGIN_DTYPE_OPTIONS</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="ow">in</span> <span class="n">PLUGIN_DTYPE_OPTIONS_MAP</span><span class="p">:</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="n">PLUGIN_DTYPE_OPTIONS_MAP</span><span class="p">[</span><span class="n">field_name</span><span class="p">]</span>
|
||||
<span class="n">annotation</span> <span class="o">=</span> <span class="n">field_info</span><span class="o">.</span><span class="n">annotation</span>
|
||||
|
||||
<span class="c1"># Extract choices from the Optional[Literal[...]] type</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">get_origin</span><span class="p">(</span><span class="n">annotation</span><span class="p">)</span> <span class="ow">is</span> <span class="n">Union</span><span class="p">:</span>
|
||||
<span class="n">args</span> <span class="o">=</span> <span class="n">get_args</span><span class="p">(</span><span class="n">annotation</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">get_origin</span><span class="p">(</span><span class="n">arg</span><span class="p">)</span> <span class="ow">is</span> <span class="n">Literal</span><span class="p">:</span>
|
||||
<span class="n">plugin_dtype_options</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">get_args</span><span class="p">(</span><span class="n">arg</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span> <span class="ow">in</span> <span class="n">args</span><span class="p">:</span>
|
||||
<span class="n">plugin_dtype_options</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">break</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">plugin_dtype_options</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">field_name</span> <span class="o">==</span> <span class="s2">"gemm_plugin"</span><span class="p">:</span>
|
||||
<span class="n">default</span> <span class="o">=</span> <span class="n">field</span><span class="o">.</span><span class="n">default</span>
|
||||
<span class="n">default</span> <span class="o">=</span> <span class="n">field_info</span><span class="o">.</span><span class="n">default</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">default</span> <span class="o">=</span> <span class="n">field</span><span class="o">.</span><span class="n">default</span> <span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">default</span> <span class="k">else</span> <span class="s2">"disable"</span>
|
||||
<span class="n">default</span> <span class="o">=</span> <span class="n">field_info</span><span class="o">.</span><span class="n">default</span> <span class="k">if</span> <span class="n">field_info</span><span class="o">.</span><span class="n">default</span> <span class="k">else</span> <span class="s2">"disable"</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
|
||||
<span class="s2">"--"</span> <span class="o">+</span> <span class="n">field_name</span><span class="p">,</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="n">default</span><span class="p">,</span>
|
||||
<span class="n">choices</span><span class="o">=</span><span class="p">[</span><span class="n">x</span> <span class="k">if</span> <span class="n">x</span> <span class="k">else</span> <span class="s2">"disable"</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">plugin_dtype_options</span><span class="p">],</span>
|
||||
<span class="n">help</span><span class="o">=</span><span class="n">help_message</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">field</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">elif</span> <span class="n">annotation</span> <span class="ow">is</span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span>
|
||||
<span class="s2">"--"</span> <span class="o">+</span> <span class="n">field_name</span><span class="p">,</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"enable"</span> <span class="k">if</span> <span class="n">field</span><span class="o">.</span><span class="n">default</span> <span class="k">else</span> <span class="s2">"disable"</span><span class="p">,</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="s2">"enable"</span> <span class="k">if</span> <span class="n">field_info</span><span class="o">.</span><span class="n">default</span> <span class="k">else</span> <span class="s2">"disable"</span><span class="p">,</span>
|
||||
<span class="n">choices</span><span class="o">=</span><span class="p">[</span><span class="s2">"enable"</span><span class="p">,</span> <span class="s2">"disable"</span><span class="p">],</span>
|
||||
<span class="n">help</span><span class="o">=</span><span class="n">help_message</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s2">"--"</span> <span class="o">+</span> <span class="n">field_name</span><span class="p">,</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="n">field</span><span class="o">.</span><span class="n">type</span><span class="p">,</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="n">field</span><span class="o">.</span><span class="n">default</span><span class="p">,</span>
|
||||
<span class="nb">type</span><span class="o">=</span><span class="n">annotation</span><span class="p">,</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="n">field_info</span><span class="o">.</span><span class="n">default</span><span class="p">,</span>
|
||||
<span class="n">help</span><span class="o">=</span><span class="n">help_message</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">parser</span>
|
||||
|
||||
@ -1471,9 +1391,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1088,9 +1092,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -783,7 +787,7 @@
|
||||
<span class="k">if</span> <span class="s2">"hf"</span> <span class="ow">in</span> <span class="n">model_dir</span><span class="p">:</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">LlavaOnevisionForConditionalGeneration</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">LlavaOnevisionForConditionalGeneration</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">model_dir</span><span class="p">,</span> <span class="n">torch_dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">,</span> <span class="n">device_map</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">model_dir</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">dtype</span><span class="p">,</span> <span class="n">device_map</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">language_model</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">llava.model.builder</span><span class="w"> </span><span class="kn">import</span> <span class="n">load_pretrained_model</span>
|
||||
@ -826,20 +830,20 @@
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">AutoModelForSeq2SeqLM</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForSeq2SeqLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="n">ckpt_path</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s2">"cuda"</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch_dtype</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">torch_dtype</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">elif</span> <span class="n">model_type_is_enc_dec</span><span class="p">(</span><span class="n">hf_config</span><span class="o">.</span><span class="n">model_type</span><span class="p">):</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">AutoModelForSeq2SeqLM</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForSeq2SeqLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="n">ckpt_path</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch_dtype</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">torch_dtype</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">EncDecModelWrapper</span><span class="p">(</span><span class="n">hf_model</span><span class="o">=</span><span class="n">model</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">model_cls</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="n">ckpt_path</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="n">device_map</span> <span class="k">if</span> <span class="n">device</span> <span class="o">!=</span> <span class="s2">"cpu"</span> <span class="k">else</span> <span class="s2">"cpu"</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="s2">"auto"</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">hf_config</span><span class="o">.</span><span class="n">model_type</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"llava"</span><span class="p">,</span> <span class="s2">"internvl_chat"</span><span class="p">]:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">language_model</span>
|
||||
@ -1886,9 +1890,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -674,12 +678,14 @@
|
||||
|
||||
<span class="c1"># encoder lora manager setup</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoder_model_config</span><span class="o">.</span><span class="n">lora_plugin</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">encoder_lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">encoder_lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">(</span>
|
||||
<span class="n">mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">encoder_runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">encoder_model_config</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="c1"># TODO: this is only for bart</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">encoder_lora_manager</span><span class="o">.</span><span class="n">load_from_hf</span><span class="p">(</span>
|
||||
<span class="n">model_dirs</span><span class="o">=</span><span class="n">lora_dir</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">encoder_model_config</span><span class="p">,</span>
|
||||
<span class="n">runtime_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">encoder_runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">component</span><span class="o">=</span><span class="s1">'encoder'</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@ -697,12 +703,14 @@
|
||||
|
||||
<span class="c1"># decoder lora manager setup</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">decoder_model_config</span><span class="o">.</span><span class="n">lora_plugin</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">decoder_lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">decoder_lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">(</span>
|
||||
<span class="n">mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">decoder_runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">decoder_model_config</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="c1"># TODO: this is only for bart</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">decoder_lora_manager</span><span class="o">.</span><span class="n">load_from_hf</span><span class="p">(</span>
|
||||
<span class="n">model_dirs</span><span class="o">=</span><span class="n">lora_dir</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">decoder_model_config</span><span class="p">,</span>
|
||||
<span class="n">runtime_mapping</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">decoder_runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">component</span><span class="o">=</span><span class="s1">'decoder'</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
@ -1154,9 +1162,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -538,7 +542,8 @@
|
||||
<span class="n">PoolsKVCacheManager</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm.runtime.redrafter_utils</span><span class="w"> </span><span class="kn">import</span> <span class="o">*</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">pad_vocab_size</span><span class="p">,</span> <span class="n">str_dtype_to_torch</span><span class="p">,</span> <span class="n">torch_to_numpy</span><span class="p">,</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.._utils</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">binding_layer_type_to_str</span><span class="p">,</span> <span class="n">binding_to_str_dtype</span><span class="p">,</span>
|
||||
<span class="n">pad_vocab_size</span><span class="p">,</span> <span class="n">str_dtype_to_torch</span><span class="p">,</span> <span class="n">torch_to_numpy</span><span class="p">,</span>
|
||||
<span class="n">trt_dtype_to_torch</span><span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..bindings</span><span class="w"> </span><span class="kn">import</span> <span class="n">KVCacheType</span><span class="p">,</span> <span class="n">ipc_nvls_allocate</span><span class="p">,</span> <span class="n">ipc_nvls_free</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..layers</span><span class="w"> </span><span class="kn">import</span> <span class="n">LanguageAdapterConfig</span>
|
||||
@ -1154,7 +1159,45 @@
|
||||
<span class="n">num_kv_heads_per_cross_attn_layer</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">skip_cross_attn_blocks</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="c1"># language adapter</span>
|
||||
<span class="n">language_adapter_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LanguageAdapterConfig</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span></div>
|
||||
<span class="n">language_adapter_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">LanguageAdapterConfig</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<div class="viewcode-block" id="ModelConfig.from_model_config_cpp">
|
||||
<a class="viewcode-back" href="../../../legacy/python-api/tensorrt_llm.runtime.html#tensorrt_llm.runtime.ModelConfig.from_model_config_cpp">[docs]</a>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_model_config_cpp</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">model_config_cpp</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'ModelConfig'</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Create a partially initialized ModelConfig instance from a given ModelConfig CPP binding instance.</span>
|
||||
|
||||
<span class="sd"> Note that each of these classes have fields that don't exist in the other, so the created ModelConfigPython</span>
|
||||
<span class="sd"> won't have all of its fields initialized.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="bp">cls</span><span class="p">(</span>
|
||||
<span class="n">max_batch_size</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">max_batch_size</span><span class="p">,</span>
|
||||
<span class="n">max_beam_width</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">max_beam_width</span><span class="p">,</span>
|
||||
<span class="n">vocab_size</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">vocab_size</span><span class="p">,</span>
|
||||
<span class="n">num_layers</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">num_layers</span><span class="p">(),</span>
|
||||
<span class="n">num_heads</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">num_heads</span><span class="p">,</span>
|
||||
<span class="n">num_kv_heads</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">num_kv_heads</span><span class="p">(</span><span class="mi">0</span><span class="p">),</span>
|
||||
<span class="n">hidden_size</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">hidden_size</span><span class="p">,</span>
|
||||
<span class="n">remove_input_padding</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">use_packed_input</span><span class="p">,</span>
|
||||
<span class="n">kv_cache_type</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">kv_cache_type</span><span class="p">,</span>
|
||||
<span class="n">cross_attention</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">use_cross_attention</span><span class="p">,</span>
|
||||
<span class="n">head_size</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">head_size</span><span class="p">,</span>
|
||||
<span class="n">max_prompt_embedding_table_size</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span>
|
||||
<span class="n">max_prompt_embedding_table_size</span><span class="p">,</span>
|
||||
<span class="n">quant_mode</span><span class="o">=</span><span class="n">QuantMode</span><span class="p">(</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">quant_mode</span><span class="o">.</span><span class="n">value</span><span class="p">),</span>
|
||||
<span class="n">gather_context_logits</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">compute_context_logits</span><span class="p">,</span>
|
||||
<span class="n">gather_generation_logits</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">compute_generation_logits</span><span class="p">,</span>
|
||||
<span class="n">gpt_attention_plugin</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">use_gpt_attention_plugin</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">binding_to_str_dtype</span><span class="p">(</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">data_type</span><span class="p">),</span>
|
||||
<span class="n">num_kv_heads_per_layer</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">num_kv_heads_per_layer</span><span class="p">,</span>
|
||||
<span class="n">tokens_per_block</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">tokens_per_block</span><span class="p">,</span>
|
||||
<span class="n">lora_plugin</span><span class="o">=</span><span class="n">model_config_cpp</span><span class="o">.</span><span class="n">use_lora_plugin</span><span class="p">,</span>
|
||||
<span class="n">layer_types</span><span class="o">=</span><span class="p">[</span>
|
||||
<span class="n">binding_layer_type_to_str</span><span class="p">(</span><span class="n">lt</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">lt</span> <span class="ow">in</span> <span class="n">model_config_cpp</span><span class="o">.</span><span class="n">layer_types</span>
|
||||
<span class="p">],</span>
|
||||
<span class="p">)</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
@ -5445,9 +5488,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1101,9 +1105,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1113,11 +1117,11 @@
|
||||
<span class="n">session</span><span class="o">.</span><span class="n">runtime</span><span class="o">.</span><span class="n">_set_weight_streaming</span><span class="p">(</span><span class="n">gpu_weights_percent</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">session</span><span class="o">.</span><span class="n">use_lora_plugin</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">()</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">(</span><span class="n">mapping</span><span class="o">=</span><span class="n">runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">model_config</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">lora_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span><span class="o">.</span><span class="n">load_from_ckpt</span><span class="p">(</span><span class="n">lora_dir</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">model_config</span><span class="p">,</span>
|
||||
<span class="n">runtime_mapping</span><span class="o">=</span><span class="n">runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">ckpt_source</span><span class="o">=</span><span class="n">lora_ckpt_source</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@ -1225,11 +1229,11 @@
|
||||
<span class="n">debug_mode</span><span class="o">=</span><span class="n">debug_mode</span><span class="p">,</span>
|
||||
<span class="n">stream</span><span class="o">=</span><span class="n">stream</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">session</span><span class="o">.</span><span class="n">use_lora_plugin</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">()</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">(</span><span class="n">mapping</span><span class="o">=</span><span class="n">runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">model_config</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">lora_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span><span class="o">.</span><span class="n">load_from_ckpt</span><span class="p">(</span><span class="n">lora_dir</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">model_config</span><span class="p">,</span>
|
||||
<span class="n">runtime_mapping</span><span class="o">=</span><span class="n">runtime_mapping</span><span class="p">,</span>
|
||||
<span class="n">ckpt_source</span><span class="o">=</span><span class="n">lora_ckpt_source</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
@ -1617,9 +1621,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -530,8 +534,9 @@
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..layers</span><span class="w"> </span><span class="kn">import</span> <span class="n">MropeParams</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..logger</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..mapping</span><span class="w"> </span><span class="kn">import</span> <span class="n">Mapping</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.generation</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">LogitsProcessor</span><span class="p">,</span> <span class="n">LoraManager</span><span class="p">,</span> <span class="n">SamplingConfig</span><span class="p">,</span>
|
||||
<span class="n">StoppingCriteria</span><span class="p">)</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.generation</span><span class="w"> </span><span class="kn">import</span> <span class="n">LogitsProcessor</span><span class="p">,</span> <span class="n">LoraManager</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.generation</span><span class="w"> </span><span class="kn">import</span> <span class="n">ModelConfig</span> <span class="k">as</span> <span class="n">ModelConfigPython</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.generation</span><span class="w"> </span><span class="kn">import</span> <span class="n">SamplingConfig</span><span class="p">,</span> <span class="n">StoppingCriteria</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">.model_runner</span><span class="w"> </span><span class="kn">import</span> <span class="n">ModelRunnerMixin</span><span class="p">,</span> <span class="n">_engine_config_to_model_config</span>
|
||||
|
||||
<span class="n">_bindings_dtype_to_torch_dtype_dict</span> <span class="o">=</span> <span class="p">{</span>
|
||||
@ -779,7 +784,11 @@
|
||||
|
||||
<span class="n">engine_config</span> <span class="o">=</span> <span class="n">EngineConfig</span><span class="o">.</span><span class="n">from_json_file</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">engine_dir</span><span class="si">}</span><span class="s2">/config.json"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">model_config</span><span class="o">.</span><span class="n">use_lora_plugin</span> <span class="ow">and</span> <span class="n">rank</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">()</span>
|
||||
<span class="n">mapping</span> <span class="o">=</span> <span class="n">_world_config_to_mapping</span><span class="p">(</span><span class="n">world_config</span><span class="p">)</span>
|
||||
<span class="n">lora_manager</span> <span class="o">=</span> <span class="n">LoraManager</span><span class="p">(</span>
|
||||
<span class="n">mapping</span><span class="o">=</span><span class="n">mapping</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">ModelConfigPython</span><span class="o">.</span><span class="n">from_model_config_cpp</span><span class="p">(</span>
|
||||
<span class="n">model_config</span><span class="p">))</span>
|
||||
<span class="k">if</span> <span class="n">lora_dir</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">config_lora_dir</span> <span class="o">=</span> <span class="n">engine_config</span><span class="o">.</span><span class="n">build_config</span><span class="o">.</span><span class="n">lora_config</span><span class="o">.</span><span class="n">lora_dir</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">config_lora_dir</span><span class="p">)</span> <span class="o">></span> <span class="mi">0</span><span class="p">:</span>
|
||||
@ -794,7 +803,6 @@
|
||||
<span class="c1"># For Executor, only rank 0 can enqueue requests, and should hold all lora weights</span>
|
||||
<span class="n">lora_manager</span><span class="o">.</span><span class="n">load_from_ckpt</span><span class="p">(</span><span class="n">lora_dir</span><span class="p">,</span>
|
||||
<span class="n">model_config</span><span class="o">=</span><span class="n">runtime_model_config</span><span class="p">,</span>
|
||||
<span class="n">runtime_mapping</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">ckpt_source</span><span class="o">=</span><span class="n">lora_ckpt_source</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
|
||||
@ -1827,9 +1835,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -1200,11 +1204,10 @@
|
||||
|
||||
<span class="c1"># Phi-4-multimodal uses pytorch engine due to issues with creating TRT engine.</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model_type</span> <span class="o">==</span> <span class="s2">"phi-4-multimodal"</span><span class="p">:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s1">'cpu'</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s1">'cpu'</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">vision_model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">embed_tokens_extend</span><span class="o">.</span><span class="n">image_embed</span><span class="o">.</span><span class="n">to</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">image_newlines</span> <span class="o">=</span> <span class="p">{}</span>
|
||||
@ -1215,11 +1218,10 @@
|
||||
<span class="k">return</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">model_type</span> <span class="o">==</span> <span class="s2">"phi-3-vision"</span><span class="p">:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s1">'cpu'</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s1">'cpu'</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">vision_model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">vision_embed_tokens</span><span class="o">.</span><span class="n">to</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">device</span><span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
|
||||
|
||||
@ -1276,7 +1278,7 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">init_audio_encoder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">model_type</span> <span class="o">==</span> <span class="s2">"phi-4-multimodal"</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">AutoModelForCausalLM</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">hf_model_dir</span><span class="p">,</span>
|
||||
<span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">float16</span><span class="p">,</span>
|
||||
<span class="n">trust_remote_code</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">device_map</span><span class="o">=</span><span class="s1">'cpu'</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">audio_model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">model</span><span class="o">.</span><span class="n">embed_tokens_extend</span><span class="o">.</span><span class="n">audio_embed</span><span class="o">.</span><span class="n">to</span><span class="p">(</span>
|
||||
@ -1376,7 +1378,7 @@
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">CLIPImageProcessor</span>
|
||||
<span class="n">processor</span> <span class="o">=</span> <span class="n">CLIPImageProcessor</span><span class="o">.</span><span class="n">from_pretrained</span><span class="p">(</span>
|
||||
<span class="s2">"openai/clip-vit-large-patch14"</span><span class="p">,</span> <span class="n">torch_dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">bfloat16</span><span class="p">)</span>
|
||||
<span class="s2">"openai/clip-vit-large-patch14"</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">bfloat16</span><span class="p">)</span>
|
||||
<span class="n">frames</span> <span class="o">=</span> <span class="n">processor</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">frames</span><span class="p">,</span>
|
||||
<span class="n">return_tensors</span><span class="o">=</span><span class="s2">"pt"</span><span class="p">)[</span><span class="s1">'pixel_values'</span><span class="p">]</span>
|
||||
<span class="c1"># make dtype consistent with vision encoder</span>
|
||||
@ -3417,9 +3419,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -961,9 +965,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -58,7 +58,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc0';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc1';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -68,7 +68,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc0" />
|
||||
<meta name="docsearch:version" content="1.2.0rc1" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -330,6 +330,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
@ -360,6 +361,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama3.3-70b-on-trtllm.html">Quick Start Recipe for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-llama4-scout-on-trtllm.html">Quick Start Recipe for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-gpt-oss-on-trtllm.html">Quick Start Recipe for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../deployment-guide/quick-start-recipe-for-qwen3-next-on-trtllm.html">Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
@ -402,6 +404,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
@ -418,6 +421,7 @@
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
@ -610,19 +614,6 @@
|
||||
<span class="k">pass</span> <span class="c1"># noqa</span>
|
||||
|
||||
|
||||
<span class="nd">@dataclass</span><span class="p">(</span><span class="n">slots</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AdditionalModelOutput</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""An additional output to gather from the model.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> name (str): The name of the additional output to gather from the model.</span>
|
||||
<span class="sd"> gather_context (bool): A value indicating whether or not to gather the additional output from the context too. Defaults to False.</span>
|
||||
<span class="sd"> """</span> <span class="c1"># noqa: E501</span>
|
||||
|
||||
<span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
|
||||
<span class="n">gather_context</span><span class="p">:</span> <span class="nb">bool</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="SamplingParams">
|
||||
<a class="viewcode-back" href="../../llm-api/reference.html#tensorrt_llm.llmapi.SamplingParams">[docs]</a>
|
||||
<span class="nd">@dataclass</span><span class="p">(</span><span class="n">slots</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">kw_only</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
@ -657,13 +648,25 @@
|
||||
<span class="sd"> best_of (int, optional): Number of sequences to consider for best output. Defaults to None.</span>
|
||||
<span class="sd"> use_beam_search (bool): Whether to use beam search. Defaults to False.</span>
|
||||
|
||||
<span class="sd"> top_k (int, optional): Controls number of logits to sample from. None means using C++ runtime default 0, i.e., all logits. Defaults to None.</span>
|
||||
<span class="sd"> top_p (float, optional): Controls the top-P probability to sample from. None means using C++ runtime default 0.f. Defaults to None.</span>
|
||||
<span class="sd"> top_k (int, optional): Controls number of logits to sample from. Can assume non-negative values, where 0 means 'all logits'. Defaults to None.</span>
|
||||
<span class="sd"> The value None is treated as "not specified" in the following.</span>
|
||||
<span class="sd"> If neither temperature, top_p, nor top_k are specified, sampling is greedy.</span>
|
||||
<span class="sd"> If temperature > 0 and/or top_p < 1 are specified, sampling will proceed accordingly and top_k will default to top_k = 0.</span>
|
||||
<span class="sd"> Setting top_k = 1 results in greedy sampling.</span>
|
||||
<span class="sd"> top_p (float, optional): Controls the top-P probability to sample from. Can have values between 0 and 1. Defaults to None.</span>
|
||||
<span class="sd"> The value None is treated as "not specified" in the following.</span>
|
||||
<span class="sd"> If neither temperature, top_p, nor top_k are specified, sampling is greedy.</span>
|
||||
<span class="sd"> If temperature > 0 and/or top_k > 1 are specified, sampling will proceed accordingly and top_p will default to top_p = 1.</span>
|
||||
<span class="sd"> Setting top_p = 0 should result in greedy sampling, but is currently disallowed in the backend.</span>
|
||||
<span class="sd"> top_p_min (float, optional): Controls decay in the top-P algorithm. topPMin is lower-bound. None means using C++ runtime default 1.e-6. Defaults to None.</span>
|
||||
<span class="sd"> top_p_reset_ids (int, optional): Controls decay in the top-P algorithm. Indicates where to reset the decay. None means using C++ runtime default 1. Defaults to None.</span>
|
||||
<span class="sd"> top_p_decay (float, optional): Controls decay in the top-P algorithm. The decay value. None means using C++ runtime default 1.f. Defaults to None.</span>
|
||||
<span class="sd"> seed (int, optional): Controls the random seed used by the random number generator in sampling. None means using C++ runtime default 0. Defaults to None.</span>
|
||||
<span class="sd"> temperature (float, optional): Controls the modulation of logits when sampling new tokens. It can have values > 0.f. None means using C++ runtime default 1.0f. Defaults to None.</span>
|
||||
<span class="sd"> temperature (float, optional): Controls the modulation of logits when sampling new tokens. It can have values >= 0.f. Defaults to None.</span>
|
||||
<span class="sd"> The value None is treated as "not specified" in the following.</span>
|
||||
<span class="sd"> If neither temperature, top_p, nor top_k are specified, sampling is greedy.</span>
|
||||
<span class="sd"> If top_p < 1 and/or top_k > 1 are specified, sampling will proceed accordingly and temperature will default to temperature = 1.</span>
|
||||
<span class="sd"> Setting temperature = 0 results in greedy sampling.</span>
|
||||
<span class="sd"> min_tokens (int, optional): Lower bound on the number of tokens to generate. Values < 1 have no effect. None means using C++ runtime default 1. Defaults to None.</span>
|
||||
<span class="sd"> beam_search_diversity_rate (float, optional): Used to penalize tokens based on how often they appear in the sequence. It can have any value > 0.f. Values < 1.f encourages repetition, values > 1.f discourages it. None means using C++ runtime default 1.f. Defaults to None.</span>
|
||||
<span class="sd"> repetition_penalty (float, optional): Used to penalize tokens based on how often they appear in the sequence. It can have any value > 0.f. Values < 1.f encourages repetition, values > 1.f discourages it. None means using C++ runtime default 1.f. Defaults to None.</span>
|
||||
@ -682,7 +685,7 @@
|
||||
<span class="sd"> exclude_input_from_output (bool): Controls if output tokens in Result should include the input tokens. Defaults to True.</span>
|
||||
<span class="sd"> return_encoder_output (bool): Controls if Result should contain encoder output hidden states (for encoder-only and encoder-decoder models). Defaults to False.</span>
|
||||
<span class="sd"> return_perf_metrics (bool): Controls if Result should contain the performance metrics for this request. Defaults to False.</span>
|
||||
<span class="sd"> additional_model_outputs (List[tensorrt_llm.sampling_params.AdditionalModelOutput], optional): The additional outputs to gather from the model. Defaults to None.</span>
|
||||
<span class="sd"> additional_model_outputs (List[str], optional): The additional outputs to gather from the model. Defaults to None.</span>
|
||||
|
||||
<span class="sd"> lookahead_config (tensorrt_llm.bindings.executor.LookaheadDecodingConfig , optional): Lookahead decoding config. Defaults to None.</span>
|
||||
<span class="sd"> guided_decoding (tensorrt_llm.sampling_params.GuidedDecodingParams, optional): Guided decoding params. Defaults to None.</span>
|
||||
@ -750,7 +753,7 @@
|
||||
<span class="n">exclude_input_from_output</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="n">return_encoder_output</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">return_perf_metrics</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
<span class="n">additional_model_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">AdditionalModelOutput</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="n">additional_model_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># Used in logprobs calculation in TRT flow to drop logits early if user did not explicitly request them.</span>
|
||||
<span class="c1"># Can be deprecated after migration to PyTorch backend.</span>
|
||||
@ -799,11 +802,19 @@
|
||||
<span class="sd"> For instance, while the greedy decoding with n > 1 is capable in the</span>
|
||||
<span class="sd"> Executor class of C++ runtime, the LLM API disallows such combination.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">n</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_p</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">top_p</span> <span class="o"><</span> <span class="mi">0</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_p</span> <span class="o">></span> <span class="mi">1</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"require 0 <= top_p <= 1, got top_p=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">top_p</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"require top_k >= 0, got top_k=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">top_k</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">temperature</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"require temperature >= 0, got temperature=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">temperature</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">n</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"best_of (</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">best_of</span><span class="si">}</span><span class="s2">) cannot be less than n (</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">n</span><span class="si">}</span><span class="s2">)"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="p">(</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="o">></span> <span class="mi">1</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">best_of</span> <span class="o">></span> <span class="mi">1</span>
|
||||
<span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_greedy_decoding</span>
|
||||
<span class="ow">and</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"TLLM_ALLOW_N_GREEDY_DECODING"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="p">):</span>
|
||||
@ -827,12 +838,28 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">logprobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">logprobs</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">logprobs</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">prompt_logprobs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">prompt_logprobs</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">prompt_logprobs</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># NB: Static, because downstream code only holds instances of</span>
|
||||
<span class="c1"># bindings.SamplingConfig (not SamplingParams).</span>
|
||||
<div class="viewcode-block" id="SamplingParams.params_imply_greedy_decoding">
|
||||
<a class="viewcode-back" href="../../llm-api/reference.html#tensorrt_llm.llmapi.SamplingParams.params_imply_greedy_decoding">[docs]</a>
|
||||
<span class="nd">@staticmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">params_imply_greedy_decoding</span><span class="p">(</span>
|
||||
<span class="o">*</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="n">top_p</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="n">top_k</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span>
|
||||
<span class="p">):</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="p">(</span><span class="n">temperature</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">top_p</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">top_k</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="ow">or</span> <span class="n">top_k</span> <span class="o">==</span> <span class="mi">1</span>
|
||||
<span class="ow">or</span> <span class="n">top_p</span> <span class="o">==</span> <span class="mf">0.0</span>
|
||||
<span class="ow">or</span> <span class="n">temperature</span> <span class="o">==</span> <span class="mi">0</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="nd">@property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_greedy_decoding</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beam_search</span>
|
||||
<span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_k</span> <span class="o">==</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">top_p</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">top_p</span> <span class="o">==</span> <span class="mf">0.0</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beam_search</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">params_imply_greedy_decoding</span><span class="p">(</span>
|
||||
<span class="n">temperature</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">temperature</span><span class="p">,</span>
|
||||
<span class="n">top_p</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">top_p</span><span class="p">,</span>
|
||||
<span class="n">top_k</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">top_k</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="nd">@property</span>
|
||||
@ -981,6 +1008,12 @@
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">config_kwargs</span><span class="p">[</span><span class="s2">"return_log_probs"</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_return_log_probs</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">config_kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"additional_model_outputs"</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">config_kwargs</span><span class="p">[</span><span class="s2">"additional_model_outputs"</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
|
||||
<span class="n">tllme</span><span class="o">.</span><span class="n">AdditionalModelOutput</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">output_name</span><span class="p">,</span> <span class="n">gather_context</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">output_name</span> <span class="ow">in</span> <span class="n">config_kwargs</span><span class="p">[</span><span class="s2">"additional_model_outputs"</span><span class="p">]</span>
|
||||
<span class="p">]</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">tllme</span><span class="o">.</span><span class="n">OutputConfig</span><span class="p">(</span><span class="o">**</span><span class="n">config_kwargs</span><span class="p">)</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_get_guided_decoding_params</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">tllme</span><span class="o">.</span><span class="n">GuidedDecodingParams</span><span class="p">:</span>
|
||||
@ -1125,9 +1158,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on September 29, 2025.</p>
|
||||
<p>Last updated on October 19, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/560ded5">560ded5</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/796891b">796891b</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -0,0 +1,239 @@
|
||||
# Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)
|
||||
|
||||
This blog post is a continuation of previous posts:
|
||||
* [Scaling Expert Parallelism in TensorRT LLM (Part 1: Design and Implementation of Large-scale EP)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.md)
|
||||
* [Scaling Expert Parallelism in TensorRT LLM (Part 2: Performance Status and Optimization)](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.md)
|
||||
|
||||
In this blog post, we focus on performance optimization, diving deeper into techniques such as lower precision, network structure refactoring, and aggressive kernel fusion. We hope this analysis and optimization process brings new inspiration to your model inference optimization work.
|
||||
|
||||
*By NVIDIA TensorRT LLM Team*
|
||||
|
||||
## Table of Contents
|
||||
- [Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)](#scaling-expert-parallelism-in-tensorrt-llm-part-3-pushing-the-performance-boundary)
|
||||
- [Table of Contents](#table-of-contents)
|
||||
- [Overview](#overview)
|
||||
- [Lower precision](#lower-precision)
|
||||
- [wo GEMM FP4 quantization](#wo-gemm-fp4-quantization)
|
||||
- [Low precision `AlltoAll`](#low-precision-alltoall)
|
||||
- [FP8 context FMHA support](#fp8-context-fmha-support)
|
||||
- [Rethink network structure](#rethink-network-structure)
|
||||
- [MTP LM head tensor parallelism](#mtp-lm-head-tensor-parallelism)
|
||||
- [Context phase Q/K/V `concat` optimization](#context-phase-qkv-concat-optimization)
|
||||
- [More kernel overlap, fusion and optimization](#more-kernel-overlap-fusion-and-optimization)
|
||||
- [Overlap kernels using programmatic dependent launch (PDL)](#overlap-kernels-using-programmatic-dependent-launch-pdl)
|
||||
- [Fuse several `AlltoAll` kernels](#fuse-several-alltoall-kernels)
|
||||
- [Fuse `add` (sparse exp and shared exp) into local reduction](#fuse-add-sparse-exp-and-shared-exp-into-local-reduction)
|
||||
- [Optimize PyTorch native `copy` and `concat` using `torch.compile`](#optimize-pytorch-native-copy-and-concat-using-torchcompile)
|
||||
- [End-to-End Performance](#end-to-end-performance)
|
||||
- [Acknowledgements](#acknowledgements)
|
||||
|
||||
## Overview
|
||||
|
||||
Let's firstly take a look at how the network structure looks like before we did the optimizations, to give an overall review on how the workloads look like:
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_overview_before_opt.png" width="600">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 1: Network structure overview before optimization</em></sub></p>
|
||||
|
||||
In this third blog of our scaling Expert Parallelism (EP) series, we push the performance boundaries of large-scale EP on NVIDIA GB200 NVL72 through multiple optimization techniques. Building upon the foundation established in [part 1](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.md) and [part 2](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.md), this blog explores three key optimization pillars: **lower precision computation** (including FP4 quantization for wo GEMM, low-precision AlltoAll communication, and FP8 context FMHA), **network structure rethinking** (featuring MTP LM head tensor parallelism and context phase Q/K/V concatenation elimination), and **aggressive kernel fusion and overlap** (leveraging Programmatic Dependent Launch, fused AlltoAll operations, and torch.compile optimizations). These optimizations collectively deliver significant end-to-end performance improvements for wide-EP scenarios on NVIDIA GB200 NVL72, for DeepSeek R1 with its specialized Multi-head Latent Attention (MLA) mechanism. Each technique is carefully designed to maintain accuracy while maximizing performance, demonstrating the power of combining algorithmic innovation with deep hardware awareness.
|
||||
|
||||
## Lower precision
|
||||
|
||||
### wo GEMM FP4 quantization
|
||||
|
||||
The wo GEMM is the final linear layer within the multi-head attention block that produces the final outputs. While DeepSeek R1's MLA modifies the initial projections for keys and values, the wo GEMM operator remains a critical and standard component for finalizing the attention computation. In the term, "wo" is the abbreviation for the weight matrix for the output.
|
||||
|
||||
We've evaluated that quantizing the wo GEMM to FP4 still satisfies the accuracy requirements, maintaining a similar MTP accept rate (AR) while improving end-to-end performance. The [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) team has published checkpoints that additionally quantize the wo module in attention layers to FP4 on HuggingFace:
|
||||
* https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2
|
||||
* https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2
|
||||
|
||||
In TensorRT LLM, this is supported by [PR 6393](https://github.com/NVIDIA/TensorRT-LLM/pull/6393). To utilize the checkpoints, simply use the LLM API or `trtllm-serve` to load them. Refer to [deploy-with-tensorrt-llm](https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2#deploy-with-tensorrt-llm) for more details.
|
||||
|
||||
### Low precision `AlltoAll`
|
||||
|
||||
In wide-EP MoE, the combine phase (after experts finish FC2) performs an all-to-all to return each token’s expert outputs to its origin rank, followed by a per-token reduce over top-k experts.
|
||||
|
||||
This step is typically bandwidth-bound when FC2 outputs are in BF16 or FP16. We introduce a low-precision AlltoAll that transmits these combine payloads in NVFP4 instead of BF16/FP16, then dequantizes back on the receiver before the local reduction.
|
||||
|
||||
During combine, we temporarily quantize the per-token expert outputs to NVFP4 (e2m1 values with per-16-element E4M3 scale factors plus a global scale) inside shared memory, send the compact representation across GPUs, and dequantize back to the original dtype on the receiving side. Indices and routing-related small tensors remain in their native types.
|
||||
|
||||
Since we quantize only for transport and outputs are dequantized back to the working dtype before the per-token reduction, we observe negligible accuracy impact; tolerances comparable to a quant-dequant roundtrip are sufficient. This feature is supported by [PR 7155](https://github.com/NVIDIA/TensorRT-LLM/pull/7155) and [PR 7898](https://github.com/NVIDIA/TensorRT-LLM/pull/7898).
|
||||
|
||||
### FP8 context FMHA support
|
||||
|
||||
FP8 context FMHA is a technique that uses the FP8 data format to accelerate the FMHA/MLA computation during the context phase of a model. This combination is designed to improve TTFT and prefill throughput, particularly when processing long contexts, without significantly sacrificing accuracy.
|
||||
|
||||
In the context phase, the K and V can be stored in FP8 format, which is often referred to as FP8 KV Cache. Using FP8 KV cache can significantly save GPU memory, which is especially beneficial for long input sequences.
|
||||
However, since Q is in BF16 format, FMHA will also be performed in BF16 format, which cannot benefit from FP8 Tensor Core.
|
||||
|
||||
With FP8 context FMHA, we first quantize Q into FP8 format, which aligns with FP8 K and V, and then leverage FP8 Tensor Core for FMHA/MLA. Since the context phase is compute-bound and Tensor Core has much higher FP8 FLOPS than BF16 FLOPS, the speed-up becomes more pronounced as the input sequence length grows.
|
||||
|
||||
Since FP8 context FMHA can maintain accuracy very close to the BF16 baseline, we enable it automatically when users use FP8 KV cache on Hopper or Blackwell. This is supported by [PR 7610](https://github.com/NVIDIA/TensorRT-LLM/pull/7610) and [PR 7612](https://github.com/NVIDIA/TensorRT-LLM/pull/7612).
|
||||
|
||||
## Rethink network structure
|
||||
|
||||
### MTP LM head tensor parallelism
|
||||
|
||||
The LM (language modeling) head is responsible for converting the `hidden_states` computed by previous decode layers to `logits`. It's a linear layer with weights in the shape of `(vocab_size, hidden_size)`, outputting logits with the shape of `(batch_size, seqlen, vocab_size)`. We are primarily interested in the logits corresponding to the last token of the input sequence, so the logits will finally be `(batch_size, vocab_size)`.
|
||||
|
||||
When MTP is enabled, the number of tokens that MTP layers handle will be equal to the batch size, while the main model will handle `(1 + MTP) * batch_size` tokens, which makes the LM head computation on MTP layers easier to fall into the memory-bound range, and 256 tokens is the empirical boundary between memory-bound and math-bound. This leads to an optimization idea: if we keep the calculation memory-bound but reduce the size of weights that need to be loaded, there could be performance benefits.
|
||||
|
||||
Based on this analysis, we conducted experiments on the following scenario: a DeepSeek R1 EP32 case with attention DP and MTP-3 enabled, where the local per-rank batch size is 32. Before the optimization, there is 32-way data parallelism, so each MTP module on each rank processes 32 tokens for LM head calculation.
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_MTP_parallel_1.png" width="500">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 2: MTP LM head computation before optimization</em></sub></p>
|
||||
|
||||
In the optimization, we first perform an `AllGather` on every 4 GPUs, so that each GB200 node has all tokens prepared for the following TP4 calculation. Then, we split LM head weights on the token dimension to fit those 4 GPUs and perform 4-way TP. Afterwards, we collect the local argmax logits on each TP rank, do a round of `AllGather` to collect that, and find the global argmax logits across all TP ranks. Collecting the local argmax logits firstly helps with minimizing communication and argmax computation overheads. Finally, we split logits to guarantee correctness.
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_MTP_parallel_2.png" width="500">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 3: MTP LM head computation after applying tensor parallelism</em></sub></p>
|
||||
|
||||
*Some layers are omitted in the diagrams above to keep the example simple.*
|
||||
|
||||
Note that we can expand the TP to 8-way to utilize multi-node NVLink, as long as we still achieve performance gains from reducing weight loading time in memory-bound scenarios.
|
||||
|
||||
This feature is supported by [PR 7571](https://github.com/NVIDIA/TensorRT-LLM/pull/7571) and [PR 7891](https://github.com/NVIDIA/TensorRT-LLM/pull/7891).
|
||||
|
||||
### Context phase Q/K/V `concat` optimization
|
||||
|
||||
In the standard attention mechanism, Q/K/V are derived from the same hidden states through `GEMM_Q`/`GEMM_K`/`GEMM_V` operations, and TensorRT LLM typically merges the weights of these three GEMMs in advance, executing a single `GEMM_QKV` to obtain a large contiguous tensor QKV, which is then used as the input to the attention kernels.
|
||||
|
||||
However, DeepSeek's MLA is a special attention module where Q/K/V are obtained by applying different downsampling-upsampling processes to the hidden states. Additionally, Q and K are divided into two parts: with RoPE and without RoPE, so a contiguous QKV tensor cannot be obtained directly.
|
||||
|
||||
In the initial implementation of context MLA, due to input format constraints of the attention kernels, TensorRT LLM had to explicitly concatenate the Q/K/V tensors into one contiguous QKV tensor, resulting in extra memory and time overhead, which became more significant in wide EP scenarios.
|
||||
|
||||
Recently, we introduced a new input format for the context MLA kernels called "separate qkv". As the name implies, these attention kernels now support three separate Q/K/V tensors as direct inputs. [PR 6538](https://github.com/NVIDIA/TensorRT-LLM/pull/6538) refactors the MLA process to eliminate the need for concatenating Q/K/V, saving copy operations and significantly improving prefill latency in wide EP scenarios.
|
||||
|
||||
## More kernel overlap, fusion and optimization
|
||||
|
||||
The team has implemented aggressive kernel fusion, overlap, and optimization to reduce kernel launch overheads and overall kernel duration. This includes overlapping kernels using PDL, fusing several `AlltoAll` kernels through refactoring, fusing sparse exp and shared exp `add` into local reduction, fusing `memset` into `expandinputrow`, fusing `finalizeMoeRouting` into FC2, and removing the `swizzle` kernel after `AlltoAll`. The following three representative examples demonstrate the common ideas behind these optimizations.
|
||||
|
||||
### Overlap kernels using programmatic dependent launch (PDL)
|
||||
|
||||
The Programmatic Dependent Launch (PDL) mechanism allows a dependent secondary kernel to launch before the primary kernel it depends on in the same CUDA stream has finished executing. Refer to the [official documentation](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#programmatic-dependent-launch-and-synchronization) for more details. TensorRT LLM has been utilizing this feature to optimize end-to-end performance.
|
||||
|
||||
We have introduced this feature to the kernels used by the wide EP workflow as well. The implementation is in [PR 7977](https://github.com/NVIDIA/TensorRT-LLM/pull/7977). We inserted the `cudaTriggerProgrammaticLaunchCompletion` API with all thread blocks in the primary kernel, which signals that it's ready for the secondary kernel to launch, and then call the `cudaGridDependencySynchronize` API in the secondary kernel, which blocks until all primary kernels the secondary kernel depends on have completed and flushed results to global memory. The following example from the [official documentation](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#api-description) demonstrates how PDL is supported in TensorRT LLM, the only difference is that we inserted `cudaTriggerProgrammaticLaunchCompletion` and `cudaGridDependencySynchronize` to the same kernel so that it can both overlap with the front and subsequent kernels.
|
||||
```c
|
||||
__global__ void primary_kernel() {
|
||||
// Initial work that should finish before starting secondary kernel
|
||||
|
||||
// Trigger the secondary kernel
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
|
||||
// Work that can coincide with the secondary kernel
|
||||
}
|
||||
|
||||
__global__ void secondary_kernel()
|
||||
{
|
||||
// Independent work
|
||||
|
||||
// Will block until all primary kernels the secondary kernel is dependent on have completed and flushed results to global memory
|
||||
cudaGridDependencySynchronize();
|
||||
|
||||
// Dependent work
|
||||
}
|
||||
```
|
||||
|
||||
We have verified the accuracy after the modification to ensure that computation results are not affected by incorrect memory reads and writes. With this premise, we made those kernels overlap as much as possible for performance considerations. In TensorRT LLM, PDL can be enabled by setting the environment variable `TRTLLM_ENABLE_PDL` to `1`, and we may introduce this as an official API in the future.
|
||||
|
||||
The effect of enabling PDL can be clearly observed using [NVIDIA Nsight Systems](https://developer.nvidia.com/nsight-systems). Taking `moeComputeRouteKernel`, `computeCountAndIndiceDevice` and `computeCumsumDevice` kernels as an example, they are executed in order when disabling PDL:
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_pdloff.png" width="1000">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 4: The profiling results of disabling PDL.</em></sub></p>
|
||||
|
||||
The following profiling results show how the three kernels overlap after enabling PDL.
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_pdlon.png" width="1000">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 5: The profiling results of enabling PDL.</em></sub></p>
|
||||
|
||||
*The above profiles were generated by using commit [84d2f12](https://github.com/NVIDIA/TensorRT-LLM/tree/84d2f1281857fbb1662b14603d3123cf327ac94f) on the main branch. They may change in future versions.*
|
||||
|
||||
For tips on using Nsys to profile and analyze TensorRT LLM performance, refer to [Coordinating with NVIDIA Nsight Systems Launch](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/developer-guide/perf-analysis.md#coordinating-with-nvidia-nsight-systems-launch).
|
||||
|
||||
### Fuse several `AlltoAll` kernels
|
||||
|
||||
To better support communication fusion—including `hiddenStates` during dispatch, low-precision ScalingFactor, MoE's `tokenSelectedExpert` and scales, as well as supporting low-precision communication during dispatch and handling potential non-alignment issues in original data, we redesigned and reimplemented `AlltoAll`.
|
||||
|
||||
Taking the dispatch of four fields as an example, the data flow is shown in Figure 6.
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_alltoall_dataflow.png" width="800">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 6: The data flow of new Alltoall kernel</em></sub></p>
|
||||
|
||||
The sending process is as follows:
|
||||
- The first step loads the original data according to the data alignment in global memory, using TMA to load into shared memory as `unAlignedData`.
|
||||
- Next, in shared memory, all fields are aligned to 16-byte boundaries and different fields are concatenated together to form `alignedData`.
|
||||
- If low-precision communication is needed, the aligned data is quantized into low-precision `lowPrecisionData`. Currently, quantization is only supported for a single field.
|
||||
- Next, corresponding encoding is performed according to the protocol. For example, with LL128, each 128 bytes contains 120 bytes of valid data and 8 bytes of flags. To avoid bank conflicts during encoding in shared memory, we select different flag positions for different packets, and the final encoded data is stored in `protoPackedData+Flag`.
|
||||
- Finally, the proto-encoded `protoPackedData+Flag` is written to the remote GPU's workspace.
|
||||
|
||||
For the receiver, it only needs to check the flag at the corresponding position in the workspace to confirm whether the data is ready. If ready, the original data is decoded in the reverse manner of sending and written to the corresponding tensors.
|
||||
|
||||
Through this approach, we can support sending and receiving multiple arbitrarily aligned fields in a fused manner and support low-precision communication during the combine process. This feature was implemented in [PR 6973](https://github.com/NVIDIA/TensorRT-LLM/pull/6973).
|
||||
|
||||
### Fuse `add` (sparse exp and shared exp) into local reduction
|
||||
|
||||
To reduce the number of kernel launches and achieve better overlap at the tail of the MoE module, we've fused the shared-expert add into the local reduction kernel that aggregates top-k experts. This removes the extra add operator without increasing the reduce operator's overhead. It also achieves single write-out and lower bandwidth occupancy.
|
||||
|
||||
The optimization is compatible with NVFP4 combine without requiring any API changes and brings no accuracy impact. It was added by [PR 7422](https://github.com/NVIDIA/TensorRT-LLM/pull/7422).
|
||||
|
||||
### Optimize PyTorch native `copy` and `concat` using `torch.compile`
|
||||
|
||||
We have observed several inefficient `copy` and `concat` operations on context phase in wide EP scenarios, and one significant case is copying `k_nope` in the MLA module. As mentioned in previous section, Q and K are divided into two parts in DeepSeek MLA: with RoPE and without RoPE. In context phase, head size of nope will be 128, and that of rope will be 64, which adds up to 192 head size. However, the FMHA kernel will directly read Q and K with head size 192, which means that we have to prepare the full Q and K using `copy` and `concat`.
|
||||
|
||||
On ISL/OSL 8k/1k, batch size 1 cases, on context phase, we observed that the `copy` operation takes 306us, which is clearly suboptimal. If we try to calculate a theoretical duration, considering 8 TB/sec HBM3e bandwidth, the formula would roughly be:
|
||||
```
|
||||
( ISL 8192 * k_nope_size 128 * num_heads 128 * 2 bytes * read/write 2 ) / ( 8 TB/sec * efficiency 0.8 ) = 80 us
|
||||
```
|
||||
|
||||
To optimize the operator, we simply added `torch.compile` decorator to the operation, and the kernel duration directly drops to 107us, which is greatly reduced and already on a promising level. [PR 8044](https://github.com/NVIDIA/TensorRT-LLM/pull/8044) implemented the changes. This is an outstanding example demonstrating the power of `torch.compile`, and showing the process of analyzing and optimizing without heavily hand-crafting kernels.
|
||||
|
||||
## End-to-End Performance
|
||||
|
||||
After applying the optimizations above, the network structure is cleaner. For example, `o_proj` and `A2A tokens` now compute in lower precision, and operators like `add` of sparse‑expert and shared‑expert is now fused into the `reduction`. The optimized parts are marked in **bold**.
|
||||
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_overview_after_opt.png" width="600">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 7: Network structure overview after optimization</em></sub></p>
|
||||
|
||||
We measured one round of performance and compared it with the baseline (main branch in July). With the optimizations mentioned above, we can see a significant performance improvement.
|
||||
<div align="center">
|
||||
<figure>
|
||||
<img src="https://github.com/NVIDIA/TensorRT-LLM/raw/main/docs/source/blogs/media/tech_blog14_perf.png" width="600">
|
||||
</figure>
|
||||
</div>
|
||||
<p align="center"><sub><em>Figure 8: End-to-End Performance on Aug 31st</em></sub></p>
|
||||
|
||||
*Note: The numbers were collected on August 31st. Some optimizations mentioned above were not yet added at that time.*
|
||||
|
||||
To review how wide EP helps with Blackwell's leading inference benchmarks, also read these recent blog posts:
|
||||
* [NVIDIA Blackwell Leads on SemiAnalysis InferenceMAX™ v1 Benchmarks](https://developer.nvidia.com/blog/nvidia-blackwell-leads-on-new-semianalysis-inferencemax-benchmarks/)
|
||||
* [NVIDIA Blackwell Raises Bar in New InferenceMAX Benchmarks, Delivering Unmatched Performance and Efficiency](https://blogs.nvidia.com/blog/blackwell-inferencemax-benchmark-results/)
|
||||
|
||||
## Acknowledgements
|
||||
This is a great continuation of previous work on TensorRT-LLM wide EP and another demonstration of excellent teamwork. It stems from brilliant performance optimization ideas, solid performance analysis and benchmarking, and rapid engineering support and implementation. By sharing these experiences, we hope to help more people who are interested in deploying large-scale LLM models on NVIDIA GPUs to run AI faster.
|
||||
@ -25,7 +25,7 @@ TensorRT LLM distributes the pre-built container on [NGC Catalog](https://catalo
|
||||
You can launch the container using the following command:
|
||||
|
||||
```bash
|
||||
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc0
|
||||
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc1
|
||||
```
|
||||
|
||||
|
||||
@ -151,16 +151,44 @@ P99 E2EL (ms): 1643.44
|
||||
|
||||
### Key Metrics
|
||||
|
||||
* Median Time to First Token (TTFT)
|
||||
#### Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
* Median Time Per Output Token (TPOT)
|
||||
* The typical time required to generate each token *after* the first one.
|
||||
* Median Inter-Token Latency (ITL)
|
||||
* The typical time delay between the completion of one token and the completion of the next.
|
||||
* Median End-to-End Latency (E2EL)
|
||||
|
||||
#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL)
|
||||
* TPOT is the typical time required to generate each token *after* the first one.
|
||||
* ITL is the typical time delay between the completion of one token and the completion of the next.
|
||||
* Both TPOT and ITL ignore TTFT.
|
||||
|
||||
For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals:
|
||||
|
||||
```math
|
||||
\text{TPOT (1\ request)} = \text{Avg(ITL)} = \frac{\text{E2E\ latency} - \text{TTFT}}{\text{\#Output\ Tokens} - 1}
|
||||
```
|
||||
|
||||
Across different requests, **average TPOT** is the mean of each request's TPOT (all requests weighted equally), while **average ITL** is token-weighted (all tokens weighted equally):
|
||||
|
||||
```math
|
||||
\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N}
|
||||
```
|
||||
|
||||
```math
|
||||
\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{\#Output Tokens across requests}}
|
||||
```
|
||||
|
||||
#### End-to-End (E2E) Latency
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
* Total Token Throughput
|
||||
|
||||
#### Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
```math
|
||||
\text{Total\ TPS} = \frac{\text{\#Input\ Tokens}+\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
#### Tokens Per Second (TPS) or Output Token Throughput
|
||||
* how many output tokens the system generates each second.
|
||||
```math
|
||||
\text{TPS} = \frac{\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
## About `extra_llm_api_options`
|
||||
trtllm-serve provides `extra_llm_api_options` knob to **overwrite** the parameters specified by trtllm-serve.
|
||||
@ -267,28 +295,28 @@ python -m tensorrt_llm.serve.scripts.benchmark_serving \
|
||||
Below is some example TensorRT-LLM serving benchmark output. Your actual results may vary.
|
||||
```
|
||||
============ Serving Benchmark Result ============
|
||||
Successful requests: 1
|
||||
Benchmark duration (s): 0.83
|
||||
Total input tokens: 128
|
||||
Total generated tokens: 128
|
||||
Request throughput (req/s): 1.20
|
||||
Output token throughput (tok/s): 153.92
|
||||
Total Token throughput (tok/s): 307.85
|
||||
User throughput (tok/s): 154.15
|
||||
Mean Request AR: 0.9845
|
||||
Median Request AR: 0.9845
|
||||
Successful requests: 1
|
||||
Benchmark duration (s): 0.83
|
||||
Total input tokens: 128
|
||||
Total generated tokens: 128
|
||||
Request throughput (req/s): 1.20
|
||||
Output token throughput (tok/s): 153.92
|
||||
Total Token throughput (tok/s): 307.85
|
||||
User throughput (tok/s): 154.15
|
||||
Mean Request AR: 0.9845
|
||||
Median Request AR: 0.9845
|
||||
---------------Time to First Token----------------
|
||||
Mean TTFT (ms): 84.03
|
||||
Median TTFT (ms): 84.03
|
||||
P99 TTFT (ms): 84.03
|
||||
Mean TTFT (ms): 84.03
|
||||
Median TTFT (ms): 84.03
|
||||
P99 TTFT (ms): 84.03
|
||||
-----Time per Output Token (excl. 1st token)------
|
||||
Mean TPOT (ms): 5.88
|
||||
Median TPOT (ms): 5.88
|
||||
P99 TPOT (ms): 5.88
|
||||
Mean TPOT (ms): 5.88
|
||||
Median TPOT (ms): 5.88
|
||||
P99 TPOT (ms): 5.88
|
||||
---------------Inter-token Latency----------------
|
||||
Mean ITL (ms): 5.83
|
||||
Median ITL (ms): 5.88
|
||||
P99 ITL (ms): 6.14
|
||||
Mean ITL (ms): 5.83
|
||||
Median ITL (ms): 5.88
|
||||
P99 ITL (ms): 6.14
|
||||
==================================================
|
||||
```
|
||||
|
||||
|
||||
@ -10,3 +10,4 @@ Model Recipes
|
||||
quick-start-recipe-for-llama3.3-70b-on-trtllm.md
|
||||
quick-start-recipe-for-llama4-scout-on-trtllm.md
|
||||
quick-start-recipe-for-gpt-oss-on-trtllm.md
|
||||
quick-start-recipe-for-qwen3-next-on-trtllm.md
|
||||
|
||||
@ -22,7 +22,7 @@ The guide is intended for developers and practitioners seeking high-throughput o
|
||||
|
||||
## MoE Backend Support Matrix
|
||||
|
||||
There are multiple MOE backends inside TRT-LLM, not all of them supporting every precision on every GPUs. Here are the support matrix of the MOE backends.
|
||||
There are multiple MOE backends inside TensorRT LLM, not all of them supporting every precision on every GPUs. Here are the support matrix of the MOE backends.
|
||||
|
||||
| device | Checkpoint | Supported moe_backend |
|
||||
|----------|----------|----------|
|
||||
@ -58,9 +58,9 @@ Note:
|
||||
* The command also maps port `8000` from the container to your host so you can access the LLM API endpoint from your host
|
||||
* See the <https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags> for all the available containers. The containers published in the main branch weekly have `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support.
|
||||
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html)
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html)
|
||||
|
||||
### Creating the TRT-LLM Server config
|
||||
### Creating the TensorRT LLM Server config
|
||||
|
||||
We create a YAML configuration file /tmp/config.yml for the TensorRT LLM Server and populate it with the following recommended performance settings.
|
||||
|
||||
@ -103,15 +103,14 @@ moe_config:
|
||||
EOF
|
||||
```
|
||||
|
||||
### Launch the TRT-LLM Server
|
||||
### Launch the TensorRT LLM Server
|
||||
|
||||
Below is an example command to launch the TRT-LLM server with the DeepSeek-R1 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
Below is an example command to launch the TensorRT LLM server with the DeepSeek-R1 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
|
||||
```shell
|
||||
trtllm-serve deepseek-ai/DeepSeek-R1-0528 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--backend pytorch \
|
||||
--max_batch_size 1024 \
|
||||
--max_num_tokens 3200 \
|
||||
--max_seq_len 2048 \
|
||||
@ -141,9 +140,6 @@ These options are used directly on the command line when you start the `trtllm-s
|
||||
* **Description:** A value between `0.0` and `1.0` that specifies the fraction of free GPU memory to reserve for the KV cache after the model is loaded. Since memory usage can fluctuate, this buffer helps prevent out-of-memory (OOM) errors.
|
||||
* **Recommendation:** If you experience OOM errors, try reducing this value to `0.7` or lower.
|
||||
|
||||
#### `--backend pytorch`
|
||||
|
||||
 **Description:** Tells TensorRT LLM to use the **pytorch** backend.
|
||||
|
||||
#### `--max_batch_size`
|
||||
|
||||
@ -230,7 +226,7 @@ Refer to the wide EP [examples](https://github.com/NVIDIA/TensorRT-LLM/tree/main
|
||||
|
||||
### Basic Test
|
||||
|
||||
Start a new terminal on the host to test the TensorRT LLM server you just launched.
|
||||
Start a new terminal on the host to test the TensorRT LLM server you just launched.
|
||||
|
||||
You can query the health/readiness of the server using:
|
||||
|
||||
@ -240,7 +236,7 @@ curl -s -o /dev/null -w "Status: %{http_code}\n" "http://localhost:8000/health"
|
||||
|
||||
When the `Status: 200` code is returned, the server is ready for queries. Note that the very first query may take longer due to initialization and compilation.
|
||||
|
||||
After the TRT-LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
After the TensorRT LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
|
||||
```shell
|
||||
curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{
|
||||
@ -251,7 +247,7 @@ curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -
|
||||
}'
|
||||
```
|
||||
|
||||
Here is an example response, showing that the TRT-LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
Here is an example response, showing that the TensorRT LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
|
||||
```json
|
||||
{"id":"cmpl-e728f08114c042309efeae4df86a50ca","object":"text_completion","created":1754294810,"model":"deepseek-ai/DeepSeek-R1-0528","choices":[{"index":0,"text":" / by Megan Stine ; illustrated by John Hinderliter.\n\nBook | Gross","token_ids":null,"logprobs":null,"context_logits":null,"finish_reason":"length","stop_reason":null,"disaggregated_params":null}],"usage":{"prompt_tokens":6,"total_tokens":22,"completion_tokens":16},"prompt_token_ids":null}
|
||||
@ -318,7 +314,7 @@ Sample result in Blackwell:
|
||||
|
||||
## Benchmarking Performance
|
||||
|
||||
To benchmark the performance of your TensorRT LLM server you can leverage the built-in “benchmark\_serving.py” script. To do this first creating a wrapper [bench.sh](http://bench.sh) script.
|
||||
To benchmark the performance of your TensorRT LLM server you can leverage the built-in `benchmark_serving.py` script. To do this first creating a wrapper `bench.sh` script.
|
||||
|
||||
```shell
|
||||
cat <<EOF > bench.sh
|
||||
@ -358,7 +354,7 @@ If you want to save the results to a file add the following options.
|
||||
--result-filename "concurrency_${concurrency}.json"
|
||||
```
|
||||
|
||||
For more benchmarking options see <https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt\_llm/serve/scripts/benchmark\_serving.py>.
|
||||
For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
|
||||
Run `bench.sh` to begin a serving benchmark. This will take a long time if you run all the concurrencies mentioned in the above `bench.sh` script.
|
||||
|
||||
@ -399,13 +395,41 @@ P99 E2EL (ms): [result]
|
||||
|
||||
### Key Metrics
|
||||
|
||||
* Median Time to First Token (TTFT)
|
||||
#### Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
* Median Time Per Output Token (TPOT)
|
||||
* The typical time required to generate each token *after* the first one.
|
||||
* Median Inter-Token Latency (ITL)
|
||||
* The typical time delay between the completion of one token and the completion of the next.
|
||||
* Median End-to-End Latency (E2EL)
|
||||
|
||||
#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL)
|
||||
* TPOT is the typical time required to generate each token *after* the first one.
|
||||
* ITL is the typical time delay between the completion of one token and the completion of the next.
|
||||
* Both TPOT and ITL ignore TTFT.
|
||||
|
||||
For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals:
|
||||
|
||||
```math
|
||||
\text{TPOT (1\ request)} = \text{Avg(ITL)} = \frac{\text{E2E\ latency} - \text{TTFT}}{\text{\#Output\ Tokens} - 1}
|
||||
```
|
||||
|
||||
Across different requests, **average TPOT** is the mean of each request's TPOT (all requests weighted equally), while **average ITL** is token-weighted (all tokens weighted equally):
|
||||
|
||||
```math
|
||||
\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N}
|
||||
```
|
||||
|
||||
```math
|
||||
\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{\#Output Tokens across requests}}
|
||||
```
|
||||
|
||||
#### End-to-End (E2E) Latency
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
* Total Token Throughput
|
||||
|
||||
#### Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
```math
|
||||
\text{Total\ TPS} = \frac{\text{\#Input\ Tokens}+\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
#### Tokens Per Second (TPS) or Output Token Throughput
|
||||
* how many output tokens the system generates each second.
|
||||
```math
|
||||
\text{TPS} = \frac{\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
@ -21,7 +21,7 @@ The guide is intended for developers and practitioners seeking high-throughput o
|
||||
|
||||
## MoE Backend Support Matrix
|
||||
|
||||
There are multiple MOE backends inside TRT-LLM. Here are the support matrix of the MOE backends.
|
||||
There are multiple MOE backends inside TensorRT LLM. Here are the support matrix of the MOE backends.
|
||||
|
||||
| Device | Activation Type | MoE Weights Type | MoE Backend | Use Case |
|
||||
|------------|------------------|------------------|-------------|----------------|
|
||||
@ -56,7 +56,7 @@ Note:
|
||||
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT-LLM, the steps refer to <https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html>.
|
||||
|
||||
### Creating the TRT-LLM Server config
|
||||
### Creating the TensorRT LLM Server config
|
||||
|
||||
We create a YAML configuration file `/tmp/config.yml` for the TensorRT-LLM Server and populate it with the following recommended performance settings.
|
||||
|
||||
@ -98,15 +98,14 @@ attention_dp_config:
|
||||
EOF
|
||||
```
|
||||
|
||||
### Launch the TRT-LLM Server
|
||||
### Launch the TensorRT LLM Server
|
||||
|
||||
Below is an example command to launch the TRT-LLM server with the GPT-OSS model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
Below is an example command to launch the TensorRT LLM server with the GPT-OSS model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
|
||||
```shell
|
||||
trtllm-serve openai/gpt-oss-120b \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--backend pytorch \
|
||||
--max_batch_size 720 \
|
||||
--max_num_tokens 16384 \
|
||||
--kv_cache_free_gpu_memory_fraction 0.9 \
|
||||
@ -135,10 +134,6 @@ These options are used directly on the command line when you start the `trtllm-s
|
||||
* **Description:** A value between `0.0` and `1.0` that specifies the fraction of free GPU memory to reserve for the KV cache after the model is loaded. Since memory usage can fluctuate, this buffer helps prevent out-of-memory (OOM) errors.
|
||||
* **Recommendation:** If you experience OOM errors, try reducing this value to `0.7` or lower.
|
||||
|
||||
#### `--backend pytorch`
|
||||
|
||||
* **Description:** Tells TensorRT-LLM to use the **pytorch** backend.
|
||||
|
||||
#### `--max_batch_size`
|
||||
|
||||
* **Description:** The maximum number of user requests that can be grouped into a single batch for processing. The actual max batch size that can be achieved depends on total sequence length (input + output).
|
||||
@ -201,7 +196,7 @@ curl -s -o /dev/null -w "Status: %{http_code}\n" "http://localhost:8000/health"
|
||||
|
||||
When the `Status: 200` code is returned, the server is ready for queries. Note that the very first query may take longer due to initialization and compilation.
|
||||
|
||||
After the TRT-LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
After the TensorRT LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
|
||||
```shell
|
||||
curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{
|
||||
@ -217,7 +212,7 @@ curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/jso
|
||||
}' -w "\n"
|
||||
```
|
||||
|
||||
Here is an example response, showing that the TRT-LLM server reasons and answers the questions.
|
||||
Here is an example response, showing that the TensorRT LLM server reasons and answers the questions.
|
||||
|
||||
TODO: Use Chat Compeletions API / Responses API as the example after the PR is merged.
|
||||
|
||||
@ -238,7 +233,7 @@ TODO: Use Chat Compeletions API / Responses API as the example after the PR is m
|
||||
We use OpenAI's official evaluation tool to test the model's accuracy. For more information see [https://github.com/openai/gpt-oss/tree/main/gpt_oss/evals](gpt-oss-eval).
|
||||
With the added support of Chat Completions and Responses API in `trtllm-serve,` `gpt_oss.evals` works directly without any modifications.
|
||||
|
||||
You need to set `enable_attention_dp`, `tp_size`, `ep_size`, `max_batch_size` and `max_num_tokens` when launching the trtllm server and set `reasoning-effort` when launching evaluation in gpt-oss. Below are some reference configurations for accuracy evaluation on B200.
|
||||
You need to set `enable_attention_dp`, `tp_size`, `ep_size`, `max_batch_size` and `max_num_tokens` when launching the trtllm server and set `reasoning-effort` when launching evaluation in gpt-oss. Below are some reference configurations for accuracy evaluation on B200.
|
||||
|
||||
| **reasoning-effort** | **parallel configuration** | **max_batch_size** | **max_num_tokens** |
|
||||
|:--------------------:|:--------------------------:|:------------------:|:------------------:|
|
||||
@ -305,7 +300,7 @@ If you want to save the results to a file add the following options.
|
||||
--result-filename "concurrency_${concurrency}.json"
|
||||
```
|
||||
|
||||
For more benchmarking options see <https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt\_llm/serve/scripts/benchmark\_serving.py>.
|
||||
For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
|
||||
Run `bench.sh` to begin a serving benchmark. This will take a long time if you run all the concurrencies mentioned in the above `bench.sh` script.
|
||||
|
||||
@ -346,13 +341,41 @@ P99 E2EL (ms): [result]
|
||||
|
||||
### Key Metrics
|
||||
|
||||
* Median Time to First Token (TTFT)
|
||||
#### Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
* Median Time Per Output Token (TPOT)
|
||||
* The typical time required to generate each token *after* the first one.
|
||||
* Median Inter-Token Latency (ITL)
|
||||
* The typical time delay between the completion of one token and the completion of the next.
|
||||
* Median End-to-End Latency (E2EL)
|
||||
|
||||
#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL)
|
||||
* TPOT is the typical time required to generate each token *after* the first one.
|
||||
* ITL is the typical time delay between the completion of one token and the completion of the next.
|
||||
* Both TPOT and ITL ignore TTFT.
|
||||
|
||||
For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals:
|
||||
|
||||
```math
|
||||
\text{TPOT (1\ request)} = \text{Avg(ITL)} = \frac{\text{E2E\ latency} - \text{TTFT}}{\text{\#Output\ Tokens} - 1}
|
||||
```
|
||||
|
||||
Across different requests, **average TPOT** is the mean of each request's TPOT (all requests weighted equally), while **average ITL** is token-weighted (all tokens weighted equally):
|
||||
|
||||
```math
|
||||
\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N}
|
||||
```
|
||||
|
||||
```math
|
||||
\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{\#Output Tokens across requests}}
|
||||
```
|
||||
|
||||
#### End-to-End (E2E) Latency
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
* Total Token Throughput
|
||||
|
||||
#### Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
```math
|
||||
\text{Total\ TPS} = \frac{\text{\#Input\ Tokens}+\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
#### Tokens Per Second (TPS) or Output Token Throughput
|
||||
* how many output tokens the system generates each second.
|
||||
```math
|
||||
\text{TPS} = \frac{\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
@ -12,15 +12,15 @@ To use Llama 3.3-70B, you must first agree to Meta’s Llama 3 Community License
|
||||
|
||||
## Prerequisites
|
||||
|
||||
GPU: NVIDIA Blackwell or Hopper Architecture
|
||||
OS: Linux
|
||||
Drivers: CUDA Driver 575 or Later
|
||||
Docker with NVIDIA Container Toolkit installed
|
||||
GPU: NVIDIA Blackwell or Hopper Architecture
|
||||
OS: Linux
|
||||
Drivers: CUDA Driver 575 or Later
|
||||
Docker with NVIDIA Container Toolkit installed
|
||||
Python3 and python3-pip (Optional, for accuracy evaluation only)
|
||||
|
||||
## Models
|
||||
|
||||
* FP8 model: [Llama-3.3-70B-Instruct-FP8](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP8)
|
||||
* FP8 model: [Llama-3.3-70B-Instruct-FP8](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP8)
|
||||
* NVFP4 model: [Llama-3.3-70B-Instruct-FP4](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP4)
|
||||
|
||||
|
||||
@ -43,16 +43,16 @@ nvcr.io/nvidia/tensorrt-llm/release:1.0.0rc6 \
|
||||
/bin/bash
|
||||
```
|
||||
|
||||
Note:
|
||||
Note:
|
||||
|
||||
* You can mount additional directories and paths using the \-v \<local\_path\>:\<path\> flag if needed, such as mounting the downloaded weight paths.
|
||||
* The command mounts your user .cache directory to save the downloaded model checkpoints which are saved to \~/.cache/huggingface/hub/ by default. This prevents having to redownload the weights each time you rerun the container. If the \~/.cache directory doesn’t exist please create it using mkdir \~/.cache
|
||||
* The command also maps port **8000** from the container to your host so you can access the LLM API endpoint from your host
|
||||
* You can mount additional directories and paths using the \-v \<local\_path\>:\<path\> flag if needed, such as mounting the downloaded weight paths.
|
||||
* The command mounts your user .cache directory to save the downloaded model checkpoints which are saved to \~/.cache/huggingface/hub/ by default. This prevents having to redownload the weights each time you rerun the container. If the \~/.cache directory doesn’t exist please create it using mkdir \~/.cache
|
||||
* The command also maps port **8000** from the container to your host so you can access the LLM API endpoint from your host
|
||||
* See the [https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags) for all the available containers. The containers published in the main branch weekly have “rcN” suffix, while the monthly release with QA tests has no “rcN” suffix. Use the rc release to get the latest model and feature support.
|
||||
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html)
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html)
|
||||
|
||||
### Creating the TRT-LLM Server config
|
||||
### Creating the TensorRT LLM Server config
|
||||
|
||||
We create a YAML configuration file /tmp/config.yml for the TensorRT LLM Server and populate it with the following recommended performance settings.
|
||||
|
||||
@ -64,20 +64,19 @@ enable_attention_dp: false
|
||||
cuda_graph_config:
|
||||
enable_padding: true
|
||||
max_batch_size: 1024
|
||||
kv_cache_config:
|
||||
kv_cache_config:
|
||||
dtype: fp8
|
||||
EOF
|
||||
```
|
||||
|
||||
### Launch the TRT-LLM Server
|
||||
### Launch the TensorRT LLM Server
|
||||
|
||||
Below is an example command to launch the TRT-LLM server with the Llama-3.3-70B-Instruct-FP8 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
Below is an example command to launch the TensorRT LLM server with the Llama-3.3-70B-Instruct-FP8 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
|
||||
```shell
|
||||
trtllm-serve nvidia/Llama-3.3-70B-Instruct-FP8 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--backend pytorch \
|
||||
--max_batch_size 1024 \
|
||||
--max_num_tokens 2048 \
|
||||
--max_seq_len 2048 \
|
||||
@ -107,10 +106,6 @@ These options are used directly on the command line when you start the `trtllm-s
|
||||
|
||||
 **Recommendation:** If you experience OOM errors, try reducing this value to **0.8** or lower.
|
||||
|
||||
#### `--backend pytorch`
|
||||
|
||||
 **Description:** Tells TensorRT LLM to use the **pytorch** backend.
|
||||
|
||||
#### `--max_batch_size`
|
||||
|
||||
 **Description:** The maximum number of user requests that can be grouped into a single batch for processing.
|
||||
@ -136,7 +131,7 @@ These options provide finer control over performance and are set within a YAML f
|
||||
|
||||
 **Description**: A section for configuring the Key-Value (KV) cache.
|
||||
|
||||
 **Options**:
|
||||
 **Options**:
|
||||
|
||||
  dtype: Sets the data type for the KV cache.
|
||||
|
||||
@ -184,7 +179,7 @@ See the [TorchLlmArgs](https://nvidia.github.io/TensorRT-LLM/llm-api/reference.h
|
||||
|
||||
### Basic Test
|
||||
|
||||
Start a new terminal on the host to test the TensorRT LLM server you just launched.
|
||||
Start a new terminal on the host to test the TensorRT LLM server you just launched.
|
||||
|
||||
You can query the health/readiness of the server using:
|
||||
|
||||
@ -194,7 +189,7 @@ curl -s -o /dev/null -w "Status: %{http_code}\n" "http://localhost:8000/health"
|
||||
|
||||
When the `Status: 200` code is returned, the server is ready for queries. Note that the very first query may take longer due to initialization and compilation.
|
||||
|
||||
After the TRT-LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
After the TensorRT LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
|
||||
```shell
|
||||
curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{
|
||||
@ -205,7 +200,7 @@ curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -
|
||||
}'
|
||||
```
|
||||
|
||||
Here is an example response, showing that the TRT-LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
Here is an example response, showing that the TensorRT LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
|
||||
```json
|
||||
{"id":"cmpl-bc1393d529ce485c961d9ffee5b25d72","object":"text_completion","created":1753843963,"model":"nvidia/Llama-3.3-70B-Instruct-FP8","choices":[{"index":0,"text":" New York is a state located in the northeastern United States. It is bordered by","token_ids":null,"logprobs":null,"context_logits":null,"finish_reason":"length","stop_reason":null,"disaggregated_params":null}],"usage":{"prompt_tokens":6,"total_tokens":22,"completion_tokens":16},"prompt_token_ids":null}
|
||||
@ -213,10 +208,10 @@ Here is an example response, showing that the TRT-LLM server returns “New York
|
||||
|
||||
### Troubleshooting Tips
|
||||
|
||||
* If you encounter CUDA out-of-memory errors, try reducing max\_batch\_size or max\_seq\_len
|
||||
* Ensure your model checkpoints are compatible with the expected format
|
||||
* For performance issues, check GPU utilization with nvidia-smi while the server is running
|
||||
* If the container fails to start, verify that the NVIDIA Container Toolkit is properly installed
|
||||
* If you encounter CUDA out-of-memory errors, try reducing max\_batch\_size or max\_seq\_len
|
||||
* Ensure your model checkpoints are compatible with the expected format
|
||||
* For performance issues, check GPU utilization with nvidia-smi while the server is running
|
||||
* If the container fails to start, verify that the NVIDIA Container Toolkit is properly installed
|
||||
* For connection issues, make sure port 8000 is not being used by another application
|
||||
|
||||
### Running Evaluations to Verify Accuracy (Optional)
|
||||
@ -241,7 +236,7 @@ MODEL_PATH=nvidia/Llama-3.3-70B-Instruct-FP8
|
||||
lm_eval --model local-completions --tasks gsm8k --batch_size 256 --gen_kwargs temperature=0.0,add_special_tokens=False --num_fewshot 5 --model_args model=${MODEL_PATH},base_url=http://localhost:8000/v1/completions,num_concurrent=32,max_retries=20,tokenized_requests=False --log_samples --output_path trtllm.fp8.gsm8k
|
||||
```
|
||||
|
||||
Sample result in Blackwell.
|
||||
Sample result in Blackwell.
|
||||
|
||||
```
|
||||
|Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr|
|
||||
@ -271,7 +266,7 @@ Sample result in Blackwell
|
||||
|
||||
## Benchmarking Performance
|
||||
|
||||
To benchmark the performance of your TensorRT LLM server you can leverage the built-in “benchmark\_serving.py” script. To do this first creating a wrapper [bench.sh](http://bench.sh) script.
|
||||
To benchmark the performance of your TensorRT LLM server you can leverage the built-in `benchmark_serving.py` script. To do this first creating a wrapper `bench.sh` script.
|
||||
|
||||
```shell
|
||||
cat <<EOF > bench.sh
|
||||
@ -311,7 +306,7 @@ If you want to save the results to a file add the following options.
|
||||
--result-filename "concurrency_${concurrency}.json"
|
||||
```
|
||||
|
||||
For more benchmarking options see. [https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt\_llm/serve/scripts/benchmark\_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
|
||||
Run bench.sh to begin a serving benchmark. This will take a long time if you run all the concurrencies mentioned in the above bench.sh script.
|
||||
|
||||
@ -352,13 +347,41 @@ P99 E2EL (ms): [result]
|
||||
|
||||
### Key Metrics
|
||||
|
||||
* Median Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
* Median Time Per Output Token (TPOT)
|
||||
* The typical time required to generate each token *after* the first one.
|
||||
* Median Inter-Token Latency (ITL)
|
||||
* The typical time delay between the completion of one token and the completion of the next.
|
||||
* Median End-to-End Latency (E2EL)
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
* Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
#### Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
|
||||
#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL)
|
||||
* TPOT is the typical time required to generate each token *after* the first one.
|
||||
* ITL is the typical time delay between the completion of one token and the completion of the next.
|
||||
* Both TPOT and ITL ignore TTFT.
|
||||
|
||||
For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals:
|
||||
|
||||
```math
|
||||
\text{TPOT (1\ request)} = \text{Avg(ITL)} = \frac{\text{E2E\ latency} - \text{TTFT}}{\text{\#Output\ Tokens} - 1}
|
||||
```
|
||||
|
||||
Across different requests, **average TPOT** is the mean of each request's TPOT (all requests weighted equally), while **average ITL** is token-weighted (all tokens weighted equally):
|
||||
|
||||
```math
|
||||
\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N}
|
||||
```
|
||||
|
||||
```math
|
||||
\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{\#Output Tokens across requests}}
|
||||
```
|
||||
|
||||
#### End-to-End (E2E) Latency
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
|
||||
#### Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
```math
|
||||
\text{Total\ TPS} = \frac{\text{\#Input\ Tokens}+\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
#### Tokens Per Second (TPS) or Output Token Throughput
|
||||
* how many output tokens the system generates each second.
|
||||
```math
|
||||
\text{TPS} = \frac{\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
@ -51,7 +51,7 @@ Note:
|
||||
|
||||
If you want to use latest main branch, you can choose to build from source to install TensorRT LLM, the steps refer to [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source-linux.html)
|
||||
|
||||
### Creating the TRT-LLM Server config
|
||||
### Creating the TensorRT LLM Server config
|
||||
|
||||
We create a YAML configuration file /tmp/config.yml for the TensorRT LLM Server and populate it with the following recommended performance settings.
|
||||
|
||||
@ -68,15 +68,14 @@ kv_cache_config:
|
||||
EOF
|
||||
```
|
||||
|
||||
### Launch the TRT-LLM Server
|
||||
### Launch the TensorRT LLM Server
|
||||
|
||||
Below is an example command to launch the TRT-LLM server with the Llama-4-Scout-17B-16E-Instruct-FP8 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
Below is an example command to launch the TensorRT LLM server with the Llama-4-Scout-17B-16E-Instruct-FP8 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “Configs and Parameters” section.
|
||||
|
||||
```shell
|
||||
trtllm-serve nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--backend pytorch \
|
||||
--max_batch_size 1024 \
|
||||
--max_num_tokens 2048 \
|
||||
--max_seq_len 2048 \
|
||||
@ -106,10 +105,6 @@ These options are used directly on the command line when you start the `trtllm-s
|
||||
* **Description:** A value between `0.0` and `1.0` that specifies the fraction of free GPU memory to reserve for the KV cache after the model is loaded. Since memory usage can fluctuate, this buffer helps prevent out-of-memory (OOM) errors.
|
||||
* **Recommendation:** If you experience OOM errors, try reducing this value to `0.7` or lower.
|
||||
|
||||
#### `--backend pytorch`
|
||||
|
||||
 **Description:** Tells TensorRT LLM to use the **pytorch** backend.
|
||||
|
||||
#### `--max_batch_size`
|
||||
|
||||
* **Description:** The maximum number of user requests that can be grouped into a single batch for processing.
|
||||
@ -191,7 +186,7 @@ curl -s -o /dev/null -w "Status: %{http_code}\n" "http://localhost:8000/health"
|
||||
|
||||
When the `Status: 200` code is returned, the server is ready for queries. Note that the very first query may take longer due to initialization and compilation.
|
||||
|
||||
After the TRT-LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
After the TensorRT LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
|
||||
```shell
|
||||
curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -d '{
|
||||
@ -202,7 +197,7 @@ curl http://localhost:8000/v1/completions -H "Content-Type: application/json" -
|
||||
}'
|
||||
```
|
||||
|
||||
Here is an example response, showing that the TRT-LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
Here is an example response, showing that the TensorRT LLM server returns “New York is a state located in the northeastern United States. It is bordered by”, completing the input sequence.
|
||||
|
||||
```json
|
||||
{"id":"cmpl-bc1393d529ce485c961d9ffee5b25d72","object":"text_completion","created":1753843963,"model":"$MODEL","choices":[{"index":0,"text":" New York is a state located in the northeastern United States. It is bordered by","token_ids":null,"logprobs":null,"context_logits":null,"finish_reason":"length","stop_reason":null,"disaggregated_params":null}],"usage":{"prompt_tokens":6,"total_tokens":22,"completion_tokens":16},"prompt_token_ids":null}
|
||||
@ -304,7 +299,7 @@ If you want to save the results to a file add the following options.
|
||||
--result-filename "concurrency_${concurrency}.json"
|
||||
```
|
||||
|
||||
For more benchmarking options see <https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt\_llm/serve/scripts/benchmark\_serving.py>.
|
||||
For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
|
||||
Run bench.sh to begin a serving benchmark. This will take a long time if you run all the concurrencies mentioned in the above bench.sh script.
|
||||
|
||||
@ -345,13 +340,41 @@ P99 E2EL (ms): [result]
|
||||
|
||||
### Key Metrics
|
||||
|
||||
* Median Time to First Token (TTFT)
|
||||
#### Time to First Token (TTFT)
|
||||
* The typical time elapsed from when a request is sent until the first output token is generated.
|
||||
* Median Time Per Output Token (TPOT)
|
||||
* The typical time required to generate each token *after* the first one.
|
||||
* Median Inter-Token Latency (ITL)
|
||||
* The typical time delay between the completion of one token and the completion of the next.
|
||||
* Median End-to-End Latency (E2EL)
|
||||
|
||||
#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL)
|
||||
* TPOT is the typical time required to generate each token *after* the first one.
|
||||
* ITL is the typical time delay between the completion of one token and the completion of the next.
|
||||
* Both TPOT and ITL ignore TTFT.
|
||||
|
||||
For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals:
|
||||
|
||||
```math
|
||||
\text{TPOT (1\ request)} = \text{Avg(ITL)} = \frac{\text{E2E\ latency} - \text{TTFT}}{\text{\#Output\ Tokens} - 1}
|
||||
```
|
||||
|
||||
Across different requests, **average TPOT** is the mean of each request's TPOT (all requests weighted equally), while **average ITL** is token-weighted (all tokens weighted equally):
|
||||
|
||||
```math
|
||||
\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N}
|
||||
```
|
||||
|
||||
```math
|
||||
\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{\#Output Tokens across requests}}
|
||||
```
|
||||
|
||||
#### End-to-End (E2E) Latency
|
||||
* The typical total time from when a request is submitted until the final token of the response is received.
|
||||
* Total Token Throughput
|
||||
|
||||
#### Total Token Throughput
|
||||
* The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens.
|
||||
```math
|
||||
\text{Total\ TPS} = \frac{\text{\#Input\ Tokens}+\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
#### Tokens Per Second (TPS) or Output Token Throughput
|
||||
* how many output tokens the system generates each second.
|
||||
```math
|
||||
\text{TPS} = \frac{\text{\#Output\ Tokens}}{T_{last} - T_{first}}
|
||||
```
|
||||
|
||||
@ -0,0 +1,237 @@
|
||||
# Quick Start Recipe for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware
|
||||
|
||||
## Introduction
|
||||
|
||||
This is a functional quick-start guide for running the Qwen3-Next model on TensorRT LLM. It focuses on a working setup with recommended defaults. Additional performance optimizations and support will be rolled out in future updates.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* GPU: NVIDIA Blackwell or Hopper Architecture
|
||||
* OS: Linux
|
||||
* Drivers: CUDA Driver 575 or Later
|
||||
* Docker with NVIDIA Container Toolkit installed
|
||||
* Python3 and python3-pip (Optional, for accuracy evaluation only)
|
||||
|
||||
## Models
|
||||
|
||||
* BF16 model: [Qwen3-Next-80B-A3B-Thinking](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking)
|
||||
|
||||
## Deployment Steps
|
||||
|
||||
### Run Docker Container
|
||||
|
||||
Build and run the docker container. See the [Docker guide](../../../docker/README.md) for details.
|
||||
```
|
||||
cd TensorRT-LLM
|
||||
|
||||
make -C docker release_build IMAGE_TAG=qwen3-next-local
|
||||
|
||||
make -C docker release_run IMAGE_NAME=tensorrt_llm IMAGE_TAG=qwen3-next-local LOCAL_USER=1
|
||||
```
|
||||
|
||||
### Creating the TensorRT LLM Server config
|
||||
|
||||
We create a YAML configuration file `/tmp/config.yml` for the TensorRT LLM Server with the following content:
|
||||
|
||||
```shell
|
||||
EXTRA_LLM_API_FILE=/tmp/config.yml
|
||||
|
||||
cat << EOF > ${EXTRA_LLM_API_FILE}
|
||||
enable_attention_dp: false
|
||||
cuda_graph_config:
|
||||
enable_padding: true
|
||||
max_batch_size: 720
|
||||
moe_config:
|
||||
backend: TRTLLM
|
||||
stream_interval: 20
|
||||
num_postprocess_workers: 4
|
||||
kv_cache_config:
|
||||
enable_block_reuse: false
|
||||
free_gpu_memory_fraction: 0.6
|
||||
EOF
|
||||
```
|
||||
|
||||
|
||||
### Launch the TensorRT LLM Server
|
||||
|
||||
Below is an example command to launch the TensorRT LLM server with the Qwen3-Next model from within the container.
|
||||
|
||||
```shell
|
||||
trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--max_batch_size 16 \
|
||||
--max_num_tokens 4096 \
|
||||
--tp_size 4 \
|
||||
--pp_size 1 \
|
||||
--ep_size 4 \
|
||||
--trust_remote_code \
|
||||
--extra_llm_api_options ${EXTRA_LLM_API_FILE}
|
||||
```
|
||||
|
||||
After the server is set up, the client can now send prompt requests to the server and receive results.
|
||||
|
||||
### Configs and Parameters
|
||||
|
||||
These options are used directly on the command line when you start the `trtllm-serve` process.
|
||||
|
||||
#### `--tp_size`
|
||||
|
||||
* **Description:** Sets the **tensor-parallel size**. This should typically match the number of GPUs you intend to use for a single model instance.
|
||||
|
||||
#### `--ep_size`
|
||||
|
||||
* **Description:** Sets the **expert-parallel size** for Mixture-of-Experts (MoE) models. Like `tp_size`, this should generally match the number of GPUs you're using. This setting has no effect on non-MoE models.
|
||||
|
||||
#### `--kv_cache_free_gpu_memory_fraction`
|
||||
|
||||
* **Description:** A value between `0.0` and `1.0` that specifies the fraction of free GPU memory to reserve for the KV cache after the model is loaded. Since memory usage can fluctuate, this buffer helps prevent out-of-memory (OOM) errors.
|
||||
* **Recommendation:** If you experience OOM errors, try reducing this value to `0.7` or lower.
|
||||
|
||||
|
||||
#### `--max_batch_size`
|
||||
|
||||
* **Description:** The maximum number of user requests that can be grouped into a single batch for processing. The actual max batch size that can be achieved depends on total sequence length (input + output).
|
||||
|
||||
#### `--max_num_tokens`
|
||||
|
||||
* **Description:** The maximum total number of tokens (across all requests) allowed inside a single scheduled batch.
|
||||
|
||||
#### `--max_seq_len`
|
||||
|
||||
* **Description:** The maximum possible sequence length for a single request, including both input and generated output tokens. We won't specifically set it. It will be inferred from model config.
|
||||
|
||||
#### `--trust_remote_code`
|
||||
|
||||
* **Description:** Allows TensorRT LLM to download models and tokenizers from Hugging Face. This flag is passed directly to the Hugging Face API.
|
||||
|
||||
|
||||
#### Extra LLM API Options (YAML Configuration)
|
||||
|
||||
These options provide finer control over performance and are set within a YAML file passed to the `trtllm-serve` command via the `--extra_llm_api_options` argument.
|
||||
|
||||
#### `cuda_graph_config`
|
||||
|
||||
* **Description**: A section for configuring CUDA graphs to optimize performance.
|
||||
|
||||
* **Options**:
|
||||
|
||||
* `enable_padding`: If `"true"`, input batches are padded to the nearest `cuda_graph_batch_size`. This can significantly improve performance.
|
||||
|
||||
**Default**: `false`
|
||||
|
||||
* `max_batch_size`: Sets the maximum batch size for which a CUDA graph will be created.
|
||||
|
||||
**Default**: `0`
|
||||
|
||||
**Recommendation**: Set this to the same value as the `--max_batch_size` command-line option.
|
||||
|
||||
#### `moe_config`
|
||||
|
||||
* **Description**: Configuration for Mixture-of-Experts (MoE) models.
|
||||
|
||||
* **Options**:
|
||||
|
||||
* `backend`: The backend to use for MoE operations.
|
||||
**Default**: `CUTLASS`
|
||||
|
||||
See the [`TorchLlmArgs` class](https://nvidia.github.io/TensorRT-LLM/llm-api/reference.html#tensorrt_llm.llmapi.TorchLlmArgs) for the full list of options which can be used in the `extra_llm_api_options`.
|
||||
|
||||
## Testing API Endpoint
|
||||
|
||||
### Basic Test
|
||||
|
||||
Start a new terminal on the host to test the TensorRT LLM server you just launched.
|
||||
|
||||
You can query the health/readiness of the server using:
|
||||
|
||||
```shell
|
||||
curl -s -o /dev/null -w "Status: %{http_code}\n" "http://localhost:8000/health"
|
||||
```
|
||||
|
||||
When the `Status: 200` code is returned, the server is ready for queries. Note that the very first query may take longer due to initialization and compilation.
|
||||
|
||||
After the TensorRT LLM server is set up and shows Application startup complete, you can send requests to the server.
|
||||
|
||||
```shell
|
||||
curl http://localhost:8000/v1/chat/completions -H "Content-Type: application/json" -d '{
|
||||
"model": "Qwen/Qwen3-Next-80B-A3B-Thinking",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Where is New York?"
|
||||
}
|
||||
],
|
||||
"max_tokens": 1024,
|
||||
"top_p": 1.0
|
||||
}' -w "\n"
|
||||
```
|
||||
|
||||
Here is an example response:
|
||||
|
||||
```
|
||||
{"id":"chatcmpl-64ac201c77bf46a7a3a4eca7759b1fd8","object":"chat.completion","created":1759022940,"model":"Qwen/Qwen3-Next-80B-A3B-Thinking","choices":[{"index":0,"message":{"role":"assistant","content":"Okay, the user is asking \"Where is New York?\" Hmm, this seems straightforward but I need to be careful. New York could mean different things—maybe they're confused about the city versus the state. \n\nFirst thought: Are they a tourist planning a trip? Or maybe a student doing homework? Could even be someone国外 who's only heard \"New York\" in movies and isn't sure if it's a city or state. \n\nI should clarify both possibilities immediately. People often mix them up. Like, if someone says \"I'm going to New York\" they're probably talking about NYC, but technically New York State is bigger. \n\nLet me break it down: \n- New York City (NYC) is the famous one—Manhattan, skyscrapers, Times Square. \n- Then New York State (NY) is the whole state, which includes NYC but also upstate areas like Albany (the capital), Buffalo, and even the Adirondacks. \n\nWait, should I mention that NYC is in New York State? Yeah, that's crucial. Otherwise they might think it's two separate things. Also, where is the state located? Northeast US, borders other states like Pennsylvania, New Jersey... maybe name a few neighbors for context. \n\nOh! And the city vs state confusion is super common. Like, people say \"New York\" for the city but forget it's part of a larger state. Should I give examples? \n- \"If someone says 'hot dogs in New York' they mean NYC\" \n- But \"the New York State Police\" covers the whole state\" \n\n...Maybe add a fun fact? Like how NYC is just one city within the state? Or that the state capital isn't NYC but Albany? Yeah, that'll help clear confusion. \n\nAlso—should I ask if they meant the city or the state? Nah, better to cover both upfront. Keep it simple but precise. No jargon. They might not know terms like \"metropolitan area\" or \"boroughs,\" so explain \"boroughs\" as neighborhoods (Manhattan, Brooklyn etc.). \n\nCoordinates: 40.7°N, 74°W? Maybe overkill unless they're into geography. But for the state, roughly 42°N to 45°N latitude? Hmm, probably unnecessary. Stick to landmarks: borders Lake Erie, Atlantic coast... \n\nWait—user might be asking literally for GPS coordinates? Unlikely, but just in case I can add: \"Geographically, New York City is at approximately 40.7°N latitude and 74.0°W longitude...\" but only if the answer feels too dry otherwise. \n\nPriority: Clear, simple, correct. No assuming prior knowledge. Say \"New York can refer to...\" then split into two cases. End with \"Most people mean the city when they say 'New York' in casual talk.\" That’s the key takeaway. \n\n...Double-check: Is Albany really the capital? Yes. And NYC is the largest city but not capital. Got it. Also, New York State is the 27th largest by area? Maybe not relevant. Stick to basics. \n\nFinal structure: \n1. \"New York can mean two things: the city or the state.\" \n2. For city: \n - Where it is (Northeast US) \n - Inside New York State \n - Famous parts (Manhattan etc.) \n3. For state: \n - Larger area, includes city \n - Borders (PA, NJ, CT, MA, VT, Canada) \n - Capital is Albany (not NYC!) \n4. Clear summary: \"In everyday talk, 'New York' usually means NYC.\" \n\nYeah. That covers it without overwhelming them. No markdown, just plain English. Keep it warm—add a \"hope that helps!\" at the end.\n</think>\n\nThat's a common point of confusion! **\"New York\" can refer to two different things**, and it's important to distinguish between them:\n\n### 1. **New York City (NYC)** \n - **What it is**: The largest and most famous city in the United States, known for landmarks like Times Square, Central Park, the Statue of Liberty, and Wall Street. \n - **Where it is**: \n - Located in the **northeastern United States**. \n - Situated at the mouth of the **Hudson River**, where it meets the **Atlantic Ocean**. \n - Part of **New York State** (see below). \n - **Geographic details**: \n - Coordinates: Approximately **40.7° N latitude, 74.0° W longitude**. \n - Composed of **5 boroughs**: Manhattan (the \"city\" most people picture), Brooklyn, Queens, The Bronx, and Staten Island. \n - Panoramic view of NYC (including Brooklyn and New Jersey skyline):","reasoning_content":null,"reasoning":null,"tool_calls":[]},"logprobs":null,"finish_reason":"length","stop_reason":null,"mm_embedding_handle":null,"disaggregated_params":null,"avg_decoded_tokens_per_iter":1.0}],"usage":{"prompt_tokens":15,"total_tokens":1039,"completion_tokens":1024},"prompt_token_ids":null}
|
||||
```
|
||||
|
||||
### Troubleshooting Tips
|
||||
|
||||
* If you encounter CUDA out-of-memory errors, try reducing `max_batch_size` or `max_seq_len`.
|
||||
* Ensure your model checkpoints are compatible with the expected format.
|
||||
* For performance issues, check GPU utilization with nvidia-smi while the server is running.
|
||||
* If the container fails to start, verify that the NVIDIA Container Toolkit is properly installed.
|
||||
* For connection issues, make sure the server port (`8000` in this guide) is not being used by another application.
|
||||
|
||||
|
||||
|
||||
## Benchmarking Performance
|
||||
|
||||
To benchmark the performance of your TensorRT LLM server you can leverage the built-in `benchmark_serving.py` script. To do this first creating a wrapper `bench.sh` script.
|
||||
|
||||
```shell
|
||||
cat <<'EOF' > bench.sh
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
concurrency_list="1 2 4 8 16 32 64 128 256"
|
||||
multi_round=5
|
||||
isl=1024
|
||||
osl=1024
|
||||
result_dir=/tmp/qwen3_output
|
||||
|
||||
for concurrency in ${concurrency_list}; do
|
||||
num_prompts=$((concurrency * multi_round))
|
||||
python -m tensorrt_llm.serve.scripts.benchmark_serving \
|
||||
--model Qwen/Qwen3-Next-80B-A3B-Thinking \
|
||||
--backend openai \
|
||||
--dataset-name "random" \
|
||||
--random-input-len ${isl} \
|
||||
--random-output-len ${osl} \
|
||||
--random-prefix-len 0 \
|
||||
--random-ids \
|
||||
--num-prompts ${num_prompts} \
|
||||
--max-concurrency ${concurrency} \
|
||||
--ignore-eos \
|
||||
--tokenize-on-client \
|
||||
--percentile-metrics "ttft,tpot,itl,e2el"
|
||||
done
|
||||
EOF
|
||||
chmod +x bench.sh
|
||||
```
|
||||
|
||||
To achieve max through-put, with attention DP on, one needs to sweep up to `concurrency = max_batch_size * num_gpus`.
|
||||
|
||||
If you want to save the results to a file add the following options.
|
||||
|
||||
```shell
|
||||
--save-result \
|
||||
--result-dir "${result_dir}" \
|
||||
--result-filename "concurrency_${concurrency}.json"
|
||||
```
|
||||
|
||||
For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py)
|
||||
|
||||
Run `bench.sh` to begin a serving benchmark. This will take a long time if you run all the concurrencies mentioned in the above `bench.sh` script.
|
||||
|
||||
```shell
|
||||
./bench.sh
|
||||
```
|
||||
@ -2,7 +2,7 @@ Curl Chat Client
|
||||
================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/curl_chat_client.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/curl_chat_client.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/curl_chat_client.sh
|
||||
:lines: 1-11
|
||||
|
||||
@ -2,7 +2,7 @@ Curl Chat Client For Multimodal
|
||||
===============================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/curl_chat_client_for_multimodal.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/curl_chat_client_for_multimodal.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/curl_chat_client_for_multimodal.sh
|
||||
:lines: 1-88
|
||||
|
||||
@ -2,7 +2,7 @@ Curl Completion Client
|
||||
======================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/curl_completion_client.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/curl_completion_client.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/curl_completion_client.sh
|
||||
:lines: 1-10
|
||||
|
||||
@ -2,7 +2,7 @@ Deepseek R1 Reasoning Parser
|
||||
============================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/deepseek_r1_reasoning_parser.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/deepseek_r1_reasoning_parser.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/deepseek_r1_reasoning_parser.sh
|
||||
:lines: 1-10
|
||||
|
||||
@ -2,7 +2,7 @@ Genai Perf Client
|
||||
=================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/genai_perf_client.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/genai_perf_client.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/genai_perf_client.sh
|
||||
:lines: 1-16
|
||||
|
||||
@ -2,7 +2,7 @@ Genai Perf Client For Multimodal
|
||||
================================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/serve/genai_perf_client_for_multimodal.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/serve/genai_perf_client_for_multimodal.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/genai_perf_client_for_multimodal.sh
|
||||
:lines: 1-19
|
||||
|
||||
@ -21,6 +21,7 @@ _____________
|
||||
llm_guided_decoding
|
||||
llm_logits_processor
|
||||
llm_multilora
|
||||
llm_sparse_attention
|
||||
llm_speculative_decoding
|
||||
llm_kv_cache_connector
|
||||
llm_kv_cache_offloading
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Generate text with guided decoding
|
||||
==================================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_guided_decoding.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_guided_decoding.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_guided_decoding.py
|
||||
:lines: 4-47
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Generate text
|
||||
=============
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_inference.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_inference.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_inference.py
|
||||
:lines: 4-35
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Generate text asynchronously
|
||||
============================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_inference_async.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_inference_async.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_inference_async.py
|
||||
:lines: 4-43
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Generate text in streaming
|
||||
==========================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_inference_async_streaming.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_inference_async_streaming.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_inference_async_streaming.py
|
||||
:lines: 4-64
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Distributed LLM Generation
|
||||
==========================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_inference_distributed.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_inference_distributed.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_inference_distributed.py
|
||||
:lines: 4-44
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
KV Cache Connector
|
||||
==================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_kv_cache_connector.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_kv_cache_connector.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_kv_cache_connector.py
|
||||
:lines: 4-247
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
KV Cache Offloading
|
||||
===================
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/560ded5450b79efde720162fc397d7efa59aae6d/examples/llm-api/llm_kv_cache_offloading.py.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/796891ba2a6959bad58c0da9645416c7264349e9/examples/llm-api/llm_kv_cache_offloading.py.
|
||||
|
||||
.. literalinclude:: ../../../examples/llm-api/llm_kv_cache_offloading.py
|
||||
:lines: 4-134
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user