mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-02-07 03:31:58 +08:00
Update latest GitHub pages to v1.2.0rc6
This commit is contained in:
Kaiyu Xie
parent
51ab35ed57
commit
2af4947777
@ -1,4 +1,4 @@
|
||||
# Sphinx build info version 1
|
||||
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
|
||||
config: e432c3509163ef03323e39d8537d99ca
|
||||
config: 370ff5f62df7a02937391c16812e12e3
|
||||
tags: 645f666f9bcd5a90fca523b33c5a78b7
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -74,7 +74,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -358,6 +358,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -366,6 +367,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1310,11 +1312,6 @@
|
||||
<span id="_CPPv3N12tensorrt_llm8executor9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm8executor9TensorPtrE"></span><span class="target" id="types_8h_1a32a3846eb7d506ec2f4699f052f54dda"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9TensorPtrE" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp type">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10SizeType32E">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm8executor10SizeType32E"></span><span class="target" id="types_8h_1ad818c2e487265ea3ec0ddd760b768085"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp type">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10SizeType64E">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor10SizeType64E"></span><span id="_CPPv2N12tensorrt_llm8executor10SizeType64E"></span><span class="target" id="types_8h_1acda8a22d5fd4b8f6f92ce04c779cf088"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E" title="Link to this definition">#</a><br /></dt>
|
||||
@ -3144,6 +3141,11 @@
|
||||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::getCommStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1a1891e3f7d95d10d503768aa993b6debf"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getCommState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp function">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager9isRunningEv">
|
||||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache17ConnectionManager9isRunningEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache17ConnectionManager9isRunningEv"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::isRunningC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1ab3ba71ff7909d1460d7086ac34e2064e"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isRunning</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager9isRunningEv" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
</div>
|
||||
</dd></dl>
|
||||
|
||||
@ -3943,6 +3945,16 @@
|
||||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||||
<dd><div class="breathe-sectiondef docutils container">
|
||||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
|
||||
<dl class="cpp type">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10SizeType32E">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm8executor10SizeType32E"></span><span id="tensorrt_llm::executor::SizeType32"></span><span class="target" id="executor_8h_1ac776d7ebaecdc4148fadc5c154f15901"></span><span class="k"><span class="pre">typedef</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp type">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5MmKeyE">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor5MmKeyE"></span><span id="_CPPv2N12tensorrt_llm8executor5MmKeyE"></span><span class="target" id="executor_8h_1a7ae0b1ae480fc64635877ec4a3477e61"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MmKey</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">array</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">uint8_t</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">32</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5MmKeyE" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
<dl class="cpp type">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17RetentionPriorityE">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor17RetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor17RetentionPriorityE"></span><span class="target" id="executor_8h_1a7d47a118ea2835238c34ba65f7ac692e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RetentionPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||||
@ -6921,8 +6933,8 @@
|
||||
<div class="breathe-sectiondef docutils container">
|
||||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||||
<dl class="cpp function">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::KVCacheStoredBlockData__IdType.tensorrt_llm::runtime::VecUniqueTokens.std::optional:tensorrt_llm::runtime::LoraTaskIdType:.SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1af6cc9927cdb952318da4d2eb2cf6eb31"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></span><span class="sig-paren">(</span>
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32NSt6vectorI5MmKeyEE">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32NSt6vectorI5MmKeyEE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32NSt6vectorI5MmKeyEE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::KVCacheStoredBlockData__IdType.tensorrt_llm::runtime::VecUniqueTokens.std::optional:tensorrt_llm::runtime::LoraTaskIdType:.SizeType32.SizeType32.std::vector:MmKey:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1a4397fd1bbe33809586fc5da7df6c9402"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></span><span class="sig-paren">(</span>
|
||||
|
||||
<dl>
|
||||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockHash</span></span></em>,</dd>
|
||||
@ -6930,9 +6942,10 @@
|
||||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE" title="tensorrt_llm::runtime::LoraTaskIdType"><span class="n"><span class="pre">LoraTaskIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">loraId</span></span></em>,</dd>
|
||||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cacheLevel</span></span></em>,</dd>
|
||||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span></em>,</dd>
|
||||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5MmKeyE" title="tensorrt_llm::executor::MmKey"><span class="n"><span class="pre">MmKey</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">mmKeys</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em>,</dd>
|
||||
</dl>
|
||||
|
||||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32NSt6vectorI5MmKeyEE" title="Link to this definition">#</a><br /></dt>
|
||||
<dd></dd></dl>
|
||||
|
||||
</div>
|
||||
@ -6968,6 +6981,12 @@
|
||||
<dd><p>The priority of the block. </p>
|
||||
</dd></dl>
|
||||
|
||||
<dl class="cpp var">
|
||||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6mmKeysE">
|
||||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData6mmKeysE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData6mmKeysE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::mmKeys__std::vector:MmKey:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1a51df5f1ec916092eded38a2e0202f717"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5MmKeyE" title="tensorrt_llm::executor::MmKey"><span class="n"><span class="pre">MmKey</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mmKeys</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6mmKeysE" title="Link to this definition">#</a><br /></dt>
|
||||
<dd><p>The multimodal keys of the block. </p>
|
||||
</dd></dl>
|
||||
|
||||
</div>
|
||||
</dd></dl>
|
||||
|
||||
@ -12245,7 +12264,6 @@
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#types-h">types.h</a><ul class="nav section-nav flex-column">
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E"><code class="docutils literal notranslate"><span class="pre">SizeType64</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE"><code class="docutils literal notranslate"><span class="pre">FloatType</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE"><code class="docutils literal notranslate"><span class="pre">TokenIdType</span></code></a></li>
|
||||
@ -12619,6 +12637,7 @@
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t"><code class="docutils literal notranslate"><span class="pre">recvConnect()</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState"><code class="docutils literal notranslate"><span class="pre">getConnections()</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"><code class="docutils literal notranslate"><span class="pre">getCommState()</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager9isRunningEv"><code class="docutils literal notranslate"><span class="pre">isRunning()</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::DataContext</span></code></a><ul class="nav section-nav flex-column">
|
||||
@ -12729,6 +12748,8 @@
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::batch_manager::kv_cache_manager</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5MmKeyE"><code class="docutils literal notranslate"><span class="pre">MmKey</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">RetentionPriority</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE"><code class="docutils literal notranslate"><span class="pre">KVCacheEventData</span></code></a></li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7versionEv"><code class="docutils literal notranslate"><span class="pre">version()</span></code></a></li>
|
||||
@ -13177,12 +13198,13 @@
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheStoredBlockData</span></code></a><ul class="nav section-nav flex-column">
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">KVCacheStoredBlockData()</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32NSt6vectorI5MmKeyEE"><code class="docutils literal notranslate"><span class="pre">KVCacheStoredBlockData()</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE"><code class="docutils literal notranslate"><span class="pre">blockHash</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE"><code class="docutils literal notranslate"><span class="pre">tokens</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE"><code class="docutils literal notranslate"><span class="pre">loraId</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE"><code class="docutils literal notranslate"><span class="pre">cacheLevel</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE"><code class="docutils literal notranslate"><span class="pre">priority</span></code></a></li>
|
||||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6mmKeysE"><code class="docutils literal notranslate"><span class="pre">mmKeys</span></code></a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheStoredData</span></code></a><ul class="nav section-nav flex-column">
|
||||
@ -13989,9 +14011,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -61,7 +61,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -74,7 +74,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -358,6 +358,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -366,6 +367,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -14701,9 +14703,9 @@ one more than decoding draft tokens for prediction from primary head </p>
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -985,6 +985,14 @@ class MLA(nn.Module):
|
||||
is_neox=pos_embd_params.is_neox,
|
||||
)
|
||||
|
||||
self.llama_4_scaling = False
|
||||
if hasattr(config.pretrained_config, 'llama_4_scaling'):
|
||||
self.llama_4_scaling = True
|
||||
self.floor_scale = getattr(config.pretrained_config.llama_4_scaling,
|
||||
'original_max_position_embeddings', 8192)
|
||||
self.attn_scale = getattr(config.pretrained_config.llama_4_scaling,
|
||||
'beta', 0.1)
|
||||
|
||||
if not config.skip_create_weights_in_init:
|
||||
self.create_weights()
|
||||
|
||||
@ -1127,6 +1135,18 @@ class MLA(nn.Module):
|
||||
return hidden_states.new_empty([num_tokens, hidden_size],
|
||||
dtype=hidden_states.dtype)
|
||||
|
||||
def _attention_scaling(self, q, position_ids):
|
||||
|
||||
def _get_attn_scale(position_ids: torch.Tensor) -> torch.Tensor:
|
||||
positions = position_ids.view(-1)
|
||||
floor = torch.floor((positions + 1.0) / self.floor_scale)
|
||||
attn_scale = torch.log(floor + 1.0) * self.attn_scale + 1.0
|
||||
return attn_scale.unsqueeze(-1)
|
||||
|
||||
attn_scale = _get_attn_scale(position_ids)
|
||||
q = (q * attn_scale).to(q.dtype)
|
||||
return q
|
||||
|
||||
def forward_impl(self,
|
||||
position_ids: Optional[torch.Tensor],
|
||||
hidden_states: torch.Tensor,
|
||||
@ -1197,6 +1217,10 @@ class MLA(nn.Module):
|
||||
assert position_ids is not None
|
||||
k_pe_ctx = self.apply_rope(q_ctx, k_pe_ctx, position_ids)
|
||||
|
||||
if self.llama_4_scaling:
|
||||
q_ctx = self._attention_scaling(
|
||||
q_ctx, position_ids[..., :num_ctx_tokens])
|
||||
|
||||
self.forward_context(
|
||||
q_ctx,
|
||||
compressed_kv_ctx,
|
||||
@ -1217,6 +1241,10 @@ class MLA(nn.Module):
|
||||
assert position_ids is not None
|
||||
k_pe_gen = self.apply_rope(q_gen, k_pe_gen, position_ids)
|
||||
|
||||
if self.llama_4_scaling:
|
||||
q_gen = self._attention_scaling(
|
||||
q_gen, position_ids[..., num_ctx_tokens:])
|
||||
|
||||
self.forward_absorption_generation(
|
||||
q_gen,
|
||||
compressed_kv_gen,
|
||||
|
||||
@ -48,7 +48,8 @@ from ..speculative import (SpecMetadata, get_num_extra_kv_tokens,
|
||||
get_spec_metadata,
|
||||
update_spec_config_from_model_config)
|
||||
from ..speculative.drafting_loops import BaseDraftingLoopWrapper
|
||||
from ..speculative.eagle3 import Eagle3ResourceManager, Eagle3SpecMetadata
|
||||
from ..speculative.eagle3 import (Eagle3OneModelSpecMetadata,
|
||||
Eagle3ResourceManager, Eagle3SpecMetadata)
|
||||
from ..speculative.mtp import SampleStateTensorsMTP
|
||||
from ..speculative.utils import SpecDecodingTensor
|
||||
from ..utils import (get_model_extra_attrs,
|
||||
@ -426,6 +427,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
mapping=self.mapping,
|
||||
dist=self.dist,
|
||||
kv_cache_manager_key=self.kv_cache_manager_key,
|
||||
sparse_attention_config=self.sparse_attention_config,
|
||||
)
|
||||
self.cuda_graph_runner = CUDAGraphRunner(cuda_graph_runner_config)
|
||||
|
||||
@ -568,13 +570,12 @@ class PyTorchModelEngine(ModelEngine):
|
||||
# Reset the global cuda graph dummy request to None in warmup.
|
||||
self.cuda_graph_runner.padding_dummy_request = None
|
||||
|
||||
cp_type = self.mapping.cp_config.get('cp_type', None)
|
||||
if cp_type is not None:
|
||||
if cp_type in [CpType.ULYSSES, CpType.STAR]:
|
||||
logger.info(
|
||||
"[ModelEngine::warmup] Skipping warmup for cp_type: ",
|
||||
cp_type.name)
|
||||
return
|
||||
if self.mapping.cp_size > 1:
|
||||
cp_type = self.mapping.cp_config.get("cp_type", None)
|
||||
logger.info(
|
||||
f"[ModelEngine::warmup] Skipping warmup for cp_type: {None if cp_type is None else cp_type.name}."
|
||||
)
|
||||
return
|
||||
|
||||
self._run_torch_compile_warmup(resource_manager)
|
||||
self._run_autotuner_warmup(resource_manager)
|
||||
@ -625,7 +626,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
"""Runs a forward pass to populate the autotuner cache."""
|
||||
if not self.llm_args.enable_autotuner:
|
||||
return
|
||||
|
||||
AutoTuner.get().setup_distributed_state(self.mapping, self.dist)
|
||||
logger.info("Running autotuner warmup...")
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
@ -635,8 +636,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
self.batch_size * (self.max_seq_len - 1))
|
||||
|
||||
cache_path = os.environ.get("TLLM_AUTOTUNER_CACHE_PATH", None)
|
||||
with self.no_cuda_graph(), autotune(cache_path=cache_path,
|
||||
rank=self.mapping.rank):
|
||||
with self.no_cuda_graph(), autotune(cache_path=cache_path):
|
||||
warmup_request = self._create_warmup_request(
|
||||
resource_manager, curr_max_num_tokens, 0)
|
||||
with self._release_batch_context(warmup_request,
|
||||
@ -704,31 +704,48 @@ class PyTorchModelEngine(ModelEngine):
|
||||
draft_lengths.append(0)
|
||||
draft_lengths = [self.max_total_draft_tokens]
|
||||
|
||||
# Create CUDA graphs for short and long sequences separately for sparse attention.
|
||||
sparse_config = self.sparse_attention_config
|
||||
if sparse_config is not None and sparse_config.needs_separate_short_long_cuda_graphs(
|
||||
):
|
||||
# For short sequences, use the (seq_len_threshold - max_draft_len - 1) as the maximum sequence length
|
||||
# to make sure all of the past and current input tokens are within the sequence length threshold.
|
||||
# For long sequences, use the default maximum sequence length (self.max_seq_len).
|
||||
max_seq_len = sparse_config.seq_len_threshold - (
|
||||
self.max_draft_len + 1)
|
||||
if max_seq_len < self.max_seq_len:
|
||||
max_seq_len_list = [self.max_seq_len, max_seq_len]
|
||||
else:
|
||||
max_seq_len_list = [self.max_seq_len]
|
||||
else:
|
||||
max_seq_len_list = [self.max_seq_len]
|
||||
|
||||
for bs in cuda_graph_batch_sizes:
|
||||
if bs > self.batch_size:
|
||||
continue
|
||||
|
||||
for draft_len in draft_lengths:
|
||||
warmup_request = self._create_cuda_graph_warmup_request(
|
||||
resource_manager, bs, draft_len)
|
||||
with self._release_batch_context(warmup_request,
|
||||
resource_manager) as batch:
|
||||
if batch is None:
|
||||
# No KV cache space, cannot continue capturing graphs
|
||||
return
|
||||
for max_seq_len in max_seq_len_list:
|
||||
warmup_request = self._create_cuda_graph_warmup_request(
|
||||
resource_manager, bs, draft_len, max_seq_len)
|
||||
with self._release_batch_context(warmup_request,
|
||||
resource_manager) as batch:
|
||||
if batch is None:
|
||||
# No KV cache space, cannot continue capturing graphs
|
||||
return
|
||||
|
||||
logger.info(
|
||||
f"Run generation-only CUDA graph warmup for batch size={bs}, draft_len={draft_len}"
|
||||
)
|
||||
logger.info(
|
||||
f"Run generation-only CUDA graph warmup for batch size={bs}, draft_len={draft_len}, max_seq_len={max_seq_len}"
|
||||
)
|
||||
|
||||
self.enable_spec_decode = draft_len > 0 or self.is_draft_model
|
||||
self._update_draft_inference_state_for_warmup(
|
||||
batch, draft_len > 0, resource_manager)
|
||||
self.enable_spec_decode = draft_len > 0 or self.is_draft_model
|
||||
self._update_draft_inference_state_for_warmup(
|
||||
batch, draft_len > 0, resource_manager)
|
||||
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
self.forward(batch,
|
||||
new_tensors_device=None,
|
||||
resource_manager=resource_manager)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
def _capture_piecewise_cuda_graphs(self, resource_manager: ResourceManager):
|
||||
"""Captures piecewise CUDA graphs for context/prefill steps via torch.compile."""
|
||||
@ -873,8 +890,11 @@ class PyTorchModelEngine(ModelEngine):
|
||||
return result
|
||||
|
||||
def _create_cuda_graph_warmup_request(
|
||||
self, resource_manager: ResourceManager, batch_size: int,
|
||||
draft_len: int) -> Optional[ScheduledRequests]:
|
||||
self,
|
||||
resource_manager: ResourceManager,
|
||||
batch_size: int,
|
||||
draft_len: int,
|
||||
max_seq_len: int = None) -> Optional[ScheduledRequests]:
|
||||
"""Creates a dummy ScheduledRequests tailored for CUDA graph capture."""
|
||||
kv_cache_manager = resource_manager.get_resource_manager(
|
||||
self.kv_cache_manager_key)
|
||||
@ -902,7 +922,8 @@ class PyTorchModelEngine(ModelEngine):
|
||||
available_tokens = kv_cache_manager.get_num_available_tokens(draft_len)
|
||||
|
||||
# Add one dummy request with the maximum possible sequence length.
|
||||
token_num = max(1, min(available_tokens, self.max_seq_len - 1))
|
||||
max_seq_len = self.max_seq_len if max_seq_len is None else max_seq_len
|
||||
token_num = max(1, min(available_tokens, max_seq_len - 1))
|
||||
model_config = self.model.model_config.pretrained_config
|
||||
max_position_embeddings = getattr(model_config,
|
||||
'max_position_embeddings', None)
|
||||
@ -1671,12 +1692,12 @@ class PyTorchModelEngine(ModelEngine):
|
||||
# Warmup doesn't have `total_input_len_cp` set because merge_helix_requests is not called.
|
||||
if not self.is_warmup and not request.is_cuda_graph_dummy:
|
||||
position_id = request.total_input_len_cp + request.py_decoding_iter - 1
|
||||
# TODO: [TRTLLM-5972] Lift the limitation that last rank is always the active one for helix.
|
||||
if self.mapping.cp_rank == self.mapping.cp_size - 1:
|
||||
past_seen_token_num = request.orig_prompt_len + request.py_decoding_iter - 1
|
||||
if request.py_helix_is_inactive_rank:
|
||||
past_seen_token_num = request.seqlen_this_rank_cp
|
||||
else:
|
||||
# past_seen_token_num doesn't grow on inactive ranks.
|
||||
past_seen_token_num = request.orig_prompt_len
|
||||
# Discount the token added to active rank in resource manager as it hasn't
|
||||
# been previously seen.
|
||||
past_seen_token_num = request.seqlen_this_rank_cp - 1
|
||||
|
||||
position_ids.append(position_id)
|
||||
num_cached_tokens_per_seq.append(past_seen_token_num)
|
||||
@ -2015,6 +2036,11 @@ class PyTorchModelEngine(ModelEngine):
|
||||
|
||||
attn_metadata.request_ids = request_ids
|
||||
attn_metadata.prompt_lens = prompt_lengths
|
||||
if helix_is_inactive_rank is not None and len(
|
||||
helix_is_inactive_rank) > 0:
|
||||
helix_is_inactive_rank = torch.tensor(helix_is_inactive_rank,
|
||||
dtype=torch.bool,
|
||||
device='cuda')
|
||||
attn_metadata.helix_is_inactive_rank = helix_is_inactive_rank
|
||||
attn_metadata.num_contexts = len(scheduled_requests.context_requests)
|
||||
# Use num_chunked_ctx_requests to record the number of extend context requests,
|
||||
@ -2089,6 +2115,9 @@ class PyTorchModelEngine(ModelEngine):
|
||||
num_accepted_draft_tokens)]
|
||||
if isinstance(spec_metadata, Eagle3SpecMetadata):
|
||||
spec_metadata.request_accepted_path = request_accepted_path
|
||||
if isinstance(spec_metadata, Eagle3OneModelSpecMetadata):
|
||||
spec_metadata.populate_sampling_params_for_one_model(
|
||||
scheduled_requests.all_requests())
|
||||
spec_metadata.prepare()
|
||||
inputs['spec_metadata'] = spec_metadata
|
||||
|
||||
@ -2643,7 +2672,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
# attn_metadata now depends on spec_metadata since it determines the shape/content of spec_dec parameter Tensors
|
||||
is_spec_dec_mode = spec_metadata.spec_dec_mode.attention_need_spec_dec_mode(
|
||||
spec_resource_manager, self.is_draft_model, self.attn_backend,
|
||||
self.model_is_wrapped, spec_metadata.is_spec_dec_tree)
|
||||
self.model_is_wrapped)
|
||||
attn_metadata.update_spec_dec_param(
|
||||
batch_size=scheduled_requests.batch_size,
|
||||
is_spec_decoding_enabled=is_spec_dec_mode,
|
||||
@ -2685,6 +2714,7 @@ class PyTorchModelEngine(ModelEngine):
|
||||
spec_metadata=spec_metadata,
|
||||
draft_tokens_cuda=self.draft_tokens_cuda
|
||||
if self.is_spec_decode else None,
|
||||
new_tensors_device=new_tensors_device,
|
||||
spec_resource_manager=spec_resource_manager,
|
||||
)
|
||||
can_run_graph = key is not None
|
||||
@ -2844,11 +2874,17 @@ class PyTorchModelEngine(ModelEngine):
|
||||
# Disable UB for unsupported platforms
|
||||
if not ub.ub_supported():
|
||||
return False
|
||||
use_nccl_symmetric = self.llm_args.allreduce_strategy == "NCCL_SYMMETRIC"
|
||||
ub.initialize_userbuffers_manager(
|
||||
self.mapping.tp_size, self.mapping.pp_size, self.mapping.cp_size,
|
||||
self.mapping.rank, self.mapping.gpus_per_node,
|
||||
hidden_size * self.max_num_tokens * 2, use_nccl_symmetric)
|
||||
# NCCL_SYMMETRIC strategy no longer requires UserBuffer allocator initialization.
|
||||
# It uses NCCLWindowAllocator from ncclUtils directly.
|
||||
if self.llm_args.allreduce_strategy == "NCCL_SYMMETRIC":
|
||||
# Skip UB initialization for NCCL_SYMMETRIC - it uses NCCLWindowAllocator directly
|
||||
return False
|
||||
ub.initialize_userbuffers_manager(self.mapping.tp_size,
|
||||
self.mapping.pp_size,
|
||||
self.mapping.cp_size,
|
||||
self.mapping.rank,
|
||||
self.mapping.gpus_per_node,
|
||||
hidden_size * self.max_num_tokens * 2)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -513,7 +515,8 @@
|
||||
<article class="bd-article">
|
||||
|
||||
<h1>All modules for which code is available</h1>
|
||||
<ul><li><a href="tensorrt_llm/bindings/executor.html">tensorrt_llm.bindings.executor</a></li>
|
||||
<ul><li><a href="tensorrt_llm/_torch/async_llm.html">tensorrt_llm._torch.async_llm</a></li>
|
||||
<li><a href="tensorrt_llm/bindings/executor.html">tensorrt_llm.bindings.executor</a></li>
|
||||
<li><a href="tensorrt_llm/builder.html">tensorrt_llm.builder</a></li>
|
||||
<li><a href="tensorrt_llm/disaggregated_params.html">tensorrt_llm.disaggregated_params</a></li>
|
||||
<li><a href="tensorrt_llm/executor/request.html">tensorrt_llm.executor.request</a></li>
|
||||
@ -694,9 +697,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
770
latest/_modules/tensorrt_llm/_torch/async_llm.html
Normal file
770
latest/_modules/tensorrt_llm/_torch/async_llm.html
Normal file
@ -0,0 +1,770 @@
|
||||
|
||||
|
||||
<!DOCTYPE html>
|
||||
|
||||
|
||||
<html lang="en" data-content_root="../../../" >
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>tensorrt_llm._torch.async_llm — TensorRT LLM</title>
|
||||
|
||||
|
||||
|
||||
<script data-cfasync="false">
|
||||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||||
</script>
|
||||
<!--
|
||||
this give us a css class that will be invisible only if js is disabled
|
||||
-->
|
||||
<noscript>
|
||||
<style>
|
||||
.pst-js-only { display: none !important; }
|
||||
|
||||
</style>
|
||||
</noscript>
|
||||
|
||||
<!-- Loaded before other Sphinx assets -->
|
||||
<link href="../../../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
|
||||
<link href="../../../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
|
||||
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=8f2a1f02" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/styles/nvidia-sphinx-theme.css?v=933278ad" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/copybutton.css?v=76b2166b" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/autodoc_pydantic.css" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/togglebutton.css?v=13237357" />
|
||||
<link rel="stylesheet" type="text/css" href="../../../_static/custom.css?v=19d20f17" />
|
||||
|
||||
<!-- So that users can add custom icons -->
|
||||
<script src="../../../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
|
||||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||||
<link rel="preload" as="script" href="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
|
||||
<link rel="preload" as="script" href="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
|
||||
|
||||
|
||||
|
||||
<script src="../../../_static/documentation_options.js?v=5929fcd5"></script>
|
||||
<script src="../../../_static/doctools.js?v=9a2dae69"></script>
|
||||
<script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||||
<script src="../../../_static/clipboard.min.js?v=a7894cd8"></script>
|
||||
<script src="../../../_static/copybutton.js?v=65e89d2a"></script>
|
||||
<script>let toggleHintShow = 'Click to show';</script>
|
||||
<script>let toggleHintHide = 'Click to hide';</script>
|
||||
<script>let toggleOpenOnPrint = 'true';</script>
|
||||
<script src="../../../_static/togglebutton.js?v=4a39c7ea"></script>
|
||||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||||
<script>DOCUMENTATION_OPTIONS.pagename = '_modules/tensorrt_llm/_torch/async_llm';</script>
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
|
||||
<link rel="icon" href="../../../_static/favicon.png"/>
|
||||
|
||||
<link rel="index" title="Index" href="../../../genindex.html" />
|
||||
<link rel="search" title="Search" href="../../../search.html" />
|
||||
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
|
||||
|
||||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||||
|
||||
|
||||
|
||||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||||
|
||||
|
||||
|
||||
<div id="pst-scroll-pixel-helper"></div>
|
||||
|
||||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||||
|
||||
|
||||
<dialog id="pst-search-dialog">
|
||||
|
||||
<form class="bd-search d-flex align-items-center"
|
||||
action="../../../search.html"
|
||||
method="get">
|
||||
<i class="fa-solid fa-magnifying-glass"></i>
|
||||
<input type="search"
|
||||
class="form-control"
|
||||
name="q"
|
||||
placeholder="Search the docs ..."
|
||||
aria-label="Search the docs ..."
|
||||
autocomplete="off"
|
||||
autocorrect="off"
|
||||
autocapitalize="off"
|
||||
spellcheck="false"/>
|
||||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<div class="pst-async-banner-revealer d-none">
|
||||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||||
</div>
|
||||
|
||||
|
||||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||||
<div class="bd-header__inner bd-page-width">
|
||||
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
|
||||
<span class="fa-solid fa-bars"></span>
|
||||
</button>
|
||||
|
||||
|
||||
<div class="col-lg-3 navbar-header-items__start">
|
||||
|
||||
<div class="navbar-item">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a class="navbar-brand logo" href="../../../index.html">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT LLM - Home"/>
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT LLM - Home"/>
|
||||
|
||||
|
||||
<p class="title logo__title">TensorRT LLM</p>
|
||||
|
||||
</a></div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="col-lg-9 navbar-header-items">
|
||||
|
||||
<div class="me-auto navbar-header-items__center">
|
||||
|
||||
<div class="navbar-item">
|
||||
|
||||
|
||||
<div class="version-switcher__container dropdown pst-js-only">
|
||||
<button id="pst-version-switcher-button-2"
|
||||
type="button"
|
||||
class="version-switcher__button btn btn-sm dropdown-toggle"
|
||||
data-bs-toggle="dropdown"
|
||||
aria-haspopup="listbox"
|
||||
aria-controls="pst-version-switcher-list-2"
|
||||
aria-label="Version switcher list"
|
||||
>
|
||||
Choose version <!-- this text may get changed later by javascript -->
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<div id="pst-version-switcher-list-2"
|
||||
class="version-switcher__menu dropdown-menu list-group-flush py-0"
|
||||
role="listbox" aria-labelledby="pst-version-switcher-button-2">
|
||||
<!-- dropdown will be populated by javascript on page load -->
|
||||
</div>
|
||||
</div></div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div class="navbar-header-items__end">
|
||||
|
||||
<div class="navbar-item navbar-persistent--container">
|
||||
|
||||
|
||||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="fa-solid fa-magnifying-glass"></i>
|
||||
<span class="search-button__default-text">Search</span>
|
||||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="navbar-item">
|
||||
|
||||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||||
</button></div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div class="navbar-persistent--mobile">
|
||||
|
||||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="fa-solid fa-magnifying-glass"></i>
|
||||
<span class="search-button__default-text">Search</span>
|
||||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</header>
|
||||
|
||||
|
||||
<div class="bd-container">
|
||||
<div class="bd-container__inner bd-page-width">
|
||||
|
||||
|
||||
|
||||
<dialog id="pst-primary-sidebar-modal"></dialog>
|
||||
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a class="navbar-brand logo" href="../../../index.html">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT LLM - Home"/>
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT LLM - Home"/>
|
||||
|
||||
|
||||
<p class="title logo__title">TensorRT LLM</p>
|
||||
|
||||
</a>
|
||||
|
||||
|
||||
|
||||
<div class="sidebar-header-items sidebar-primary__section">
|
||||
|
||||
|
||||
<div class="sidebar-header-items__center">
|
||||
|
||||
|
||||
|
||||
<div class="navbar-item">
|
||||
|
||||
|
||||
<div class="version-switcher__container dropdown pst-js-only">
|
||||
<button id="pst-version-switcher-button-3"
|
||||
type="button"
|
||||
class="version-switcher__button btn btn-sm dropdown-toggle"
|
||||
data-bs-toggle="dropdown"
|
||||
aria-haspopup="listbox"
|
||||
aria-controls="pst-version-switcher-list-3"
|
||||
aria-label="Version switcher list"
|
||||
>
|
||||
Choose version <!-- this text may get changed later by javascript -->
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<div id="pst-version-switcher-list-3"
|
||||
class="version-switcher__menu dropdown-menu list-group-flush py-0"
|
||||
role="listbox" aria-labelledby="pst-version-switcher-button-3">
|
||||
<!-- dropdown will be populated by javascript on page load -->
|
||||
</div>
|
||||
</div></div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="sidebar-header-items__end">
|
||||
|
||||
<div class="navbar-item">
|
||||
|
||||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||||
</button></div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||||
<div class="sidebar-primary-item">
|
||||
|
||||
|
||||
|
||||
<nav class="bd-docs-nav bd-links"
|
||||
aria-label="Table of Contents">
|
||||
<p class="bd-links__title" role="heading" aria-level="1">Table of Contents</p>
|
||||
<div class="bd-toc-item navbar-nav"><p aria-level="2" class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../overview.html">Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../quick-start-guide.html">Quick Start Guide</a></li>
|
||||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../installation/index.html">Installation</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../installation/containers.html">Pre-built release container images on NGC</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../installation/linux.html">Installing on Linux via <code class="docutils literal notranslate"><span class="pre">pip</span></code></a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../installation/build-from-source-linux.html">Building from Source Code on Linux</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deployment Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../examples/llm_api_examples.html">LLM Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_inference.html">Generate text</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_inference_async.html">Generate text asynchronously</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_inference_async_streaming.html">Generate text in streaming</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_inference_distributed.html">Distributed LLM Generation</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_runtime.html">Runtime Configuration Examples</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_sampling.html">Sampling Techniques Showcase</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_mgmn_llm_distributed.html">Run LLM-API with pytorch backend on Slurm</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_mgmn_trtllm_bench.html">Run trtllm-bench with pytorch backend on Slurm</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/llm_mgmn_trtllm_serve.html">Run trtllm-serve with pytorch backend on Slurm</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../examples/trtllm_serve_examples.html">Online Serving Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_chat_client.html">OpenAI Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_chat_client_for_multimodal.html">OpenAI Chat Client for Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../deployment-guide/index.html">Model Recipes</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html">Deployment Guide for DeepSeek R1 on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html">Deployment Guide for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html">Deployment Guide for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html">Deployment Guide for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-qwen3-on-trtllm.html">Deployment Guide for Qwen3 on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html">Deployment Guide for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../deployment-guide/deployment-guide-for-kimi-k2-thinking-on-trtllm.html">Deployment Guide for Kimi K2 Thinking on TensorRT LLM - Blackwell</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Models</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../models/supported-models.html">Supported Models</a></li>
|
||||
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../models/adding-new-model.html">Adding a New Model</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">CLI Reference</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../commands/trtllm-bench.html">trtllm-bench</a></li>
|
||||
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../commands/trtllm-eval.html">trtllm-eval</a></li>
|
||||
<li class="toctree-l1 has-children"><a class="reference internal" href="../../../commands/trtllm-serve/index.html">trtllm-serve</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../commands/trtllm-serve/trtllm-serve.html">trtllm-serve</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../commands/trtllm-serve/run-benchmark-with-trtllm-serve.html">Run benchmarking with <code class="docutils literal notranslate"><span class="pre">trtllm-serve</span></code></a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">API Reference</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../llm-api/index.html">LLM API Introduction</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../llm-api/reference.html">API Reference</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Features</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/feature-combination-matrix.html">Feature Combination Matrix</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/attention.html">Multi-Head, Multi-Query, and Group-Query Attention</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/disagg-serving.html">Disaggregated Serving</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/kvcache.html">KV Cache System</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/long-sequence.html">Long Sequences</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/lora.html">LoRA (Low-Rank Adaptation)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/multi-modality.html">Multimodal Support in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/overlap-scheduler.html">Overlap Scheduler</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/paged-attention-ifb-scheduler.html">Paged Attention, IFB, and Request Scheduling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/parallel-strategy.html">Parallelism in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/quantization.html">Quantization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/sampling.html">Sampling</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/additional-outputs.html">Additional Outputs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/guided-decoding.html">Guided Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/torch_compile_and_piecewise_cuda_graph.html">Torch Compile & Piecewise CUDA Graph</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/helix.html">Helix Parallelism</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../features/kv-cache-connector.html">KV Cache Connector</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/overview.html">Architecture Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/perf-analysis.html">Performance Analysis</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/perf-benchmarking.html">TensorRT LLM Benchmarking</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/ci-overview.html">Continuous Integration Overview</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/dev-containers.html">Using Dev Containers</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/api-change.html">LLM API Change Guide</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../developer-guide/kv-transfer.html">Introduction to KV Cache Transmission</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Blogs</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog10_ADP_Balance_Strategy.html">ADP Balance Strategy</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html">Scaling Expert Parallelism in TensorRT LLM (Part 1: Design and Implementation of Large-scale EP)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html">Disaggregated Serving in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html">How to launch Llama4 Maverick + Eagle3 TensorRT LLM server</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html">N-Gram Speculative Decoding in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html">Scaling Expert Parallelism in TensorRT LLM (Part 2: Performance Status and Optimization)</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html">Running a High Performance GPT-OSS-120B Inference Server with TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html">How to get best performance on DeepSeek-R1 in TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/H200launch.html">H200 achieves nearly 12,000 tokens/sec on Llama2-13B with TensorRT LLM</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/XQA-kernel.html">New XQA-kernel provides 2.4x more Llama-70B throughput within the same latency budget</a></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../blogs/H100vsA100.html">H100 has 4.6x A100 Performance in TensorRT LLM, achieving 10,000 tok/s at 100ms to first token</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Quick Links</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM/releases">Releases</a></li>
|
||||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM">Github Code</a></li>
|
||||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap">Roadmap</a></li>
|
||||
</ul>
|
||||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Use TensorRT Engine</span></p>
|
||||
<ul class="nav bd-sidenav">
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../legacy/tensorrt_quickstart.html">LLM API with TensorRT Engine</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</nav></div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<main id="main-content" class="bd-main" role="main">
|
||||
|
||||
|
||||
<div class="bd-content">
|
||||
<div class="bd-article-container">
|
||||
|
||||
<div class="bd-header-article d-print-none">
|
||||
<div class="header-article-items header-article__inner">
|
||||
|
||||
<div class="header-article-items__start">
|
||||
|
||||
<div class="header-article-item">
|
||||
|
||||
<nav aria-label="Breadcrumb" class="d-print-none">
|
||||
<ul class="bd-breadcrumbs">
|
||||
|
||||
<li class="breadcrumb-item breadcrumb-home">
|
||||
<a href="../../../index.html" class="nav-link" aria-label="Home">
|
||||
<i class="fa-solid fa-home"></i>
|
||||
</a>
|
||||
</li>
|
||||
|
||||
<li class="breadcrumb-item"><a href="../../index.html" class="nav-link">Module code</a></li>
|
||||
|
||||
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">tensorrt_llm._torch.async_llm</span></li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div id="searchbox"></div>
|
||||
<article class="bd-article">
|
||||
|
||||
<h1>Source code for tensorrt_llm._torch.async_llm</h1><div class="highlight"><pre>
|
||||
<span></span><span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..llmapi.llm</span><span class="w"> </span><span class="kn">import</span> <span class="n">LLM</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">..llmapi.llm_args</span><span class="w"> </span><span class="kn">import</span> <span class="n">RayPlacementConfig</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM">[docs]</a>
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">AsyncLLM</span><span class="p">(</span><span class="n">LLM</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""AsyncLLM is a subclass of LLM that supports asynchronous setup, release and</span>
|
||||
<span class="sd"> resume operations that are necessary for RL or agentic scenarios.</span>
|
||||
|
||||
<span class="sd"> Currently, RL APIs are only supported with Ray orchestrator.</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.__init__">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.__init__">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">placement_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">placement_bundle_indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">per_worker_gpu_share</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
|
||||
<span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"orchestrator_type"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"ray"</span>
|
||||
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"ray_placement_config"</span><span class="p">]</span> <span class="o">=</span> <span class="n">RayPlacementConfig</span><span class="p">(</span>
|
||||
<span class="n">defer_workers_init</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">placement_groups</span><span class="o">=</span><span class="n">placement_groups</span><span class="p">,</span>
|
||||
<span class="n">placement_bundle_indices</span><span class="o">=</span><span class="n">placement_bundle_indices</span><span class="p">,</span>
|
||||
<span class="n">per_worker_gpu_share</span><span class="o">=</span><span class="n">per_worker_gpu_share</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># WAR: RL integration needs to use NCCL AllReduce for TP>1 due to a bug in TRTLLM's AllReduce</span>
|
||||
<span class="c1"># which will cause convergence issue when using multiple rollout instances.</span>
|
||||
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"allreduce_strategy"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"NCCL"</span>
|
||||
|
||||
<span class="k">if</span> <span class="s2">"ray_worker_extension_cls"</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
|
||||
<span class="n">kwargs</span><span class="p">[</span><span class="s2">"ray_worker_extension_cls"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"tensorrt_llm.llmapi.rlhf_utils.WorkerExtension"</span>
|
||||
|
||||
<span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_async_initialized</span> <span class="o">=</span> <span class="kc">False</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.setup_async">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.setup_async">[docs]</a>
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">setup_async</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Setup the LLM asynchronously."""</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_async_initialized</span><span class="p">:</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="o">.</span><span class="n">init_workers_async</span><span class="p">()</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="o">.</span><span class="n">setup_engine_remote_async</span><span class="p">()</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_async_initialized</span> <span class="o">=</span> <span class="kc">True</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.release">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.release">[docs]</a>
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">release</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tags</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="w"> </span><span class="sd">"""Release the GPU memory used by the LLM asynchronously.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> tags: List of memory tag strings to release (e.g., ["model", "kv_cache"]).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">collective_rpc</span><span class="p">(</span><span class="s2">"sleep"</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">tags</span><span class="p">,))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.resume">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.resume">[docs]</a>
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">resume</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tags</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="w"> </span><span class="sd">"""Resume the GPU memory used by the LLM asynchronously.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> tags: List of memory tag strings to resume (e.g., ["model", "kv_cache"]).</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">collective_rpc</span><span class="p">(</span><span class="s2">"wakeup"</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">tags</span><span class="p">,))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.update_weights">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.update_weights">[docs]</a>
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">update_weights</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">weights</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]):</span>
|
||||
<span class="w"> </span><span class="sd">"""Update the weights of the LLM asynchronously.</span>
|
||||
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> weights: Dictionary mapping device UUIDs to IPC handles for weight tensors.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">collective_rpc</span><span class="p">(</span><span class="s2">"update_weights"</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">weights</span><span class="p">,))</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="AsyncLLM.collective_rpc">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.AsyncLLM.collective_rpc">[docs]</a>
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">collective_rpc</span><span class="p">(</span>
|
||||
<span class="bp">self</span><span class="p">,</span>
|
||||
<span class="n">method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||
<span class="n">args</span><span class="p">:</span> <span class="nb">tuple</span><span class="p">[</span><span class="n">Any</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">=</span> <span class="p">(),</span>
|
||||
<span class="n">kwargs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">unique_reply_rank</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">)</span> <span class="o">-></span> <span class="nb">list</span><span class="p">[</span><span class="n">Any</span><span class="p">]:</span>
|
||||
<span class="w"> </span><span class="sd">"""Execute an asynchronous RPC call on all GPU workers. Currently, this is only supported for RayExecutor.</span>
|
||||
|
||||
<span class="sd"> Args:</span>
|
||||
<span class="sd"> method (str): The name of the worker method to execute.</span>
|
||||
<span class="sd"> args (tuple[Any, ...]): Positional arguments to pass to the worker method. Defaults to ().</span>
|
||||
<span class="sd"> kwargs (dict, optional): Keyword arguments to pass to the worker method. Defaults to None.</span>
|
||||
<span class="sd"> unique_reply_rank (int, optional): The rank of the worker that will be used to send the reply.</span>
|
||||
|
||||
<span class="sd"> Returns:</span>
|
||||
<span class="sd"> list[Any]: A list of results from each worker.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">_executor</span><span class="o">.</span><span class="n">collective_rpc_async</span><span class="p">(</span>
|
||||
<span class="n">method</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">unique_reply_rank</span><span class="o">=</span><span class="n">unique_reply_rank</span>
|
||||
<span class="p">)</span></div>
|
||||
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__await__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">setup_async</span><span class="p">()</span><span class="o">.</span><span class="fm">__await__</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">"Please use 'async with AsyncLLM' instead"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="fm">__aenter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="k">await</span> <span class="bp">self</span><span class="o">.</span><span class="n">setup_async</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__enter__</span><span class="p">()</span>
|
||||
|
||||
<span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="fm">__aexit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">):</span>
|
||||
<span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__exit__</span><span class="p">(</span><span class="n">exc_type</span><span class="p">,</span> <span class="n">exc_val</span><span class="p">,</span> <span class="n">exc_tb</span><span class="p">)</span></div>
|
||||
|
||||
</pre></div>
|
||||
|
||||
</article>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<footer class="prev-next-footer d-print-none">
|
||||
|
||||
<div class="prev-next-area">
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="bd-sidebar-secondary"></div>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
<footer class="bd-footer-content">
|
||||
|
||||
</footer>
|
||||
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||||
<script defer src="../../../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
|
||||
<script defer src="../../../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
|
||||
|
||||
|
||||
<footer class="bd-footer">
|
||||
<div class="bd-footer__inner bd-page-width">
|
||||
|
||||
<div class="footer-items__start">
|
||||
|
||||
<div class="footer-item">
|
||||
<a class="footer-brand logo" href="https://www.nvidia.com">
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-1c-blk-for-screen.svg" class="logo__image only-light" alt="NVIDIA"/>
|
||||
<img src="../../../_static/nvidia-logo-horiz-rgb-1c-wht-for-screen.svg" class="logo__image only-dark" alt="NVIDIA"/>
|
||||
</a></div>
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
<div class="footer-links">
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-policy/">Privacy Policy</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-center/">Your Privacy Choices</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/terms-of-service/">Terms of Service</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/accessibility/">Accessibility</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/company-policies/">Corporate Policies</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/product-security/">Product Security</a>
|
||||
|
|
||||
|
||||
|
||||
|
||||
<a class="external" href="https://www.nvidia.com/en-us/contact/">Contact</a>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer-item">
|
||||
|
||||
|
||||
|
||||
|
||||
<p class="copyright">
|
||||
|
||||
Copyright © 2025, NVidia.
|
||||
<br/>
|
||||
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1922,9 +1924,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -725,9 +727,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -770,9 +772,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1595,9 +1597,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -801,9 +803,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -4742,7 +4744,8 @@
|
||||
<span class="n">TWOSHOT</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">LOWPRECISION</span> <span class="o">=</span> <span class="mi">6</span>
|
||||
<span class="n">MNNVL</span> <span class="o">=</span> <span class="mi">7</span>
|
||||
<span class="n">NCCL_SYMMETRIC</span> <span class="o">=</span> <span class="mi">8</span></div>
|
||||
<span class="n">NCCL_SYMMETRIC</span> <span class="o">=</span> <span class="mi">8</span>
|
||||
<span class="n">SYMM_MEM</span> <span class="o">=</span> <span class="mi">9</span> <span class="c1"># PyTorch symmetric memory with MULTIMEM</span></div>
|
||||
|
||||
|
||||
|
||||
@ -4909,7 +4912,10 @@
|
||||
<span class="n">pfc</span> <span class="o">=</span> <span class="n">trt</span><span class="o">.</span><span class="n">PluginFieldCollection</span><span class="p">(</span><span class="n">pfc</span><span class="p">)</span>
|
||||
<span class="n">ar_plug</span> <span class="o">=</span> <span class="n">allreduce_plg_creator</span><span class="o">.</span><span class="n">create_plugin</span><span class="p">(</span><span class="s2">"allreduce"</span><span class="p">,</span> <span class="n">pfc</span><span class="p">)</span>
|
||||
<span class="n">plug_inputs</span> <span class="o">=</span> <span class="p">[</span><span class="n">tensor</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">!=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL</span> <span class="ow">and</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">!=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">UB</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span>
|
||||
<span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL</span><span class="p">,</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">UB</span><span class="p">,</span>
|
||||
<span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL_SYMMETRIC</span>
|
||||
<span class="p">}:</span>
|
||||
<span class="n">plug_inputs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">workspace</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">fusion_op</span> <span class="o">!=</span> <span class="n">AllReduceFusionOp</span><span class="o">.</span><span class="n">NONE</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">has_bias</span><span class="p">()</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||
@ -4984,7 +4990,7 @@
|
||||
<span class="n">workspace</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">!=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL</span> <span class="ow">and</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">!=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">UB</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">current_all_reduce_helper</span><span class="p">()</span><span class="o">.</span><span class="n">workspace</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL</span>
|
||||
<span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">=</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">NCCL_SYMMETRIC</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">workspace</span> <span class="o">=</span> <span class="n">current_all_reduce_helper</span><span class="p">()</span><span class="o">.</span><span class="n">workspace</span><span class="o">.</span><span class="n">trt_tensor</span>
|
||||
<span class="k">if</span> <span class="n">all_reduce_params</span><span class="o">.</span><span class="n">strategy</span> <span class="o">==</span> <span class="n">AllReduceStrategy</span><span class="o">.</span><span class="n">UB</span><span class="p">:</span>
|
||||
@ -8778,9 +8784,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -652,9 +654,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -3515,9 +3517,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -659,9 +661,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -908,9 +910,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1375,9 +1377,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1223,9 +1225,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1249,9 +1251,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1013,9 +1015,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -668,9 +670,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -951,9 +953,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -717,7 +719,7 @@
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">mpi_session</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">mpi_session</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">parallel_config</span><span class="o">.</span><span class="n">is_multi_gpu</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">get_device_count</span><span class="p">(</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">getenv</span><span class="p">(</span><span class="s2">"RAY_LOCAL_WORLD_SIZE"</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">get_device_count</span><span class="p">(</span>
|
||||
<span class="p">)</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">parallel_config</span><span class="o">.</span><span class="n">world_size_per_node</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"Only </span><span class="si">{</span><span class="n">get_device_count</span><span class="p">()</span><span class="si">}</span><span class="s2"> GPUs are available, but </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">parallel_config</span><span class="o">.</span><span class="n">world_size</span><span class="si">}</span><span class="s2"> are required."</span>
|
||||
@ -753,7 +755,6 @@
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">runtime_context</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_ModelRuntimeContext</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">llm_build_stats</span> <span class="o">=</span> <span class="n">LlmBuildStats</span><span class="p">()</span>
|
||||
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">_build_model</span><span class="p">()</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
||||
@ -1802,9 +1803,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -537,6 +539,11 @@
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">strenum</span><span class="w"> </span><span class="kn">import</span> <span class="n">StrEnum</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">transformers</span><span class="w"> </span><span class="kn">import</span> <span class="n">PreTrainedTokenizerBase</span>
|
||||
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">ray.util.placement_group</span><span class="w"> </span><span class="kn">import</span> <span class="n">PlacementGroup</span>
|
||||
<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
|
||||
<span class="n">PlacementGroup</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="kn">from</span><span class="w"> </span><span class="nn">tensorrt_llm.lora_helper</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span><span class="n">LoraConfig</span><span class="p">,</span>
|
||||
<span class="n">get_default_trtllm_modules_to_hf_modules</span><span class="p">)</span>
|
||||
|
||||
@ -707,6 +714,11 @@
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Configuration for sparse attention.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">seq_len_threshold</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"The sequence length threshold for separating short and long sequences."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">from_dict</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">):</span>
|
||||
@ -742,6 +754,15 @@
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_indices_block_size</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">needs_separate_short_long_cuda_graphs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Determines whether to capture a dedicated CUDA graph for batches consisting entirely of short sequences.</span>
|
||||
<span class="sd"> If True, capture distinct graphs for short-only batches and general cases (e.g., long or mixed batches).</span>
|
||||
<span class="sd"> If False, capture a single unified CUDA graph for all sequences regardless of length.</span>
|
||||
<span class="sd"> The seq_len_threshold parameter defines the cutoff boundary between short and long sequences.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">return</span> <span class="kc">False</span>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="RocketSparseAttentionConfig">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.RocketSparseAttentionConfig">[docs]</a>
|
||||
@ -801,6 +822,11 @@
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"The topk for the indexer."</span><span class="p">)</span>
|
||||
<span class="n">indexer_max_chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">description</span><span class="o">=</span><span class="s2">"The maximum chunk size for the indexer."</span><span class="p">)</span>
|
||||
<span class="c1"># TODO: enable this by default once the memory usage in attention metadata is optimized</span>
|
||||
<span class="n">skip_indexer_for_short_seqs</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Whether to skip the MQA and Top-K in the indexer for short sequences."</span><span class="p">)</span>
|
||||
|
||||
<div class="viewcode-block" id="DeepSeekSparseAttentionConfig.from_dict">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.DeepSeekSparseAttentionConfig.from_dict">[docs]</a>
|
||||
@ -813,6 +839,17 @@
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.DeepSeekSparseAttentionConfig.supports_backend">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">supports_backend</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">backend</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">backend</span> <span class="o">==</span> <span class="s2">"pytorch"</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="DeepSeekSparseAttentionConfig.needs_separate_short_long_cuda_graphs">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.DeepSeekSparseAttentionConfig.needs_separate_short_long_cuda_graphs">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">needs_separate_short_long_cuda_graphs</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Whether to capture separate CUDA graphs for short and long sequences.</span>
|
||||
<span class="sd"> Use seq_len_threshold to determine the threshold for separating short and long sequences.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">seq_len_threshold</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">index_topk</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">skip_indexer_for_short_seqs</span></div>
|
||||
</div>
|
||||
|
||||
|
||||
@ -1180,6 +1217,10 @@
|
||||
<span class="c1"># (N = acceptance_window) drops below this value.</span>
|
||||
<span class="n">acceptance_length_threshold</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
|
||||
|
||||
<span class="c1"># Prototype. If true, allows non-greedy sampling when speculation is used. Only applicable</span>
|
||||
<span class="c1"># to 1-model code paths; non-greedy sampling is always enabled on 2-model paths.</span>
|
||||
<span class="n">allow_advanced_sampling</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
|
||||
|
||||
<span class="c1"># Validate acceptance controls at field level so they run on model creation</span>
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s1">'acceptance_window'</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
@ -1748,6 +1789,65 @@
|
||||
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">RayPlacementConfig</span><span class="p">(</span><span class="n">StrictBaseModel</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""</span>
|
||||
<span class="sd"> Configuration for Ray GPU workers placement.</span>
|
||||
<span class="sd"> This config is only used with AsyncLLM for RL scenarios.</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">defer_workers_init</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Defer Ray worker initialization until async setup."</span><span class="p">)</span>
|
||||
|
||||
<span class="n">placement_groups</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"List of Ray placement groups, one per node. "</span>
|
||||
<span class="s2">"Each element must be a ray.util.placement_group.PlacementGroup instance."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">placement_bundle_indices</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"List of bundle indices for each placement group. "</span>
|
||||
<span class="s2">"Outer list corresponds to placement_groups, inner list contains bundle indices for that group."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">per_worker_gpu_share</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"GPU fraction per worker for colocation scenarios. "</span>
|
||||
<span class="s2">"Example: 0.1 means 10 actors can share one GPU. Defaults to 1.0 (one actor per GPU)."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="nd">@model_validator</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s1">'after'</span><span class="p">)</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate_ray_placement</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'RayPlacementConfig'</span><span class="p">:</span>
|
||||
<span class="n">has_pgs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">placement_groups</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
<span class="n">has_indices</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">placement_bundle_indices</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">has_pgs</span> <span class="o">!=</span> <span class="n">has_indices</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="s2">"placement_groups and placement_bundle_indices must be provided together"</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">has_pgs</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">placement_groups</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">placement_bundle_indices</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"placement_groups length (</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">placement_groups</span><span class="p">)</span><span class="si">}</span><span class="s2">) must equal "</span>
|
||||
<span class="sa">f</span><span class="s2">"placement_bundle_indices length (</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">placement_bundle_indices</span><span class="p">)</span><span class="si">}</span><span class="s2">)"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">PlacementGroup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">pg</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">placement_groups</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pg</span><span class="p">,</span> <span class="n">PlacementGroup</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"placement_groups[</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">] must be a Ray PlacementGroup, "</span>
|
||||
<span class="sa">f</span><span class="s2">"got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">pg</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">per_worker_gpu_share</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="mi">0</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">per_worker_gpu_share</span> <span class="o"><=</span> <span class="mf">1.0</span><span class="p">):</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"per_worker_gpu_share must be between 0 and 1.0, "</span>
|
||||
<span class="sa">f</span><span class="s2">"got </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">per_worker_gpu_share</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
|
||||
|
||||
<span class="k">class</span><span class="w"> </span><span class="nc">PybindMirror</span><span class="p">(</span><span class="n">ABC</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">''' A class containing the utilities for mirroring Python classes to</span>
|
||||
<span class="sd"> pybinding classes.</span>
|
||||
@ -2675,9 +2775,17 @@
|
||||
<span class="n">env_overrides</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won’t update unless the code fetches them from os.environ on demand."</span><span class="p">,</span>
|
||||
<span class="s2">"[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won't update unless the code fetches them from os.environ on demand."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s1">'env_overrides'</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'before'</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">coerce_env_overrides_to_str</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
|
||||
<span class="w"> </span><span class="sd">"""Coerce env_overrides values to strings for os.environ compatibility."""</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">v</span>
|
||||
<span class="k">return</span> <span class="p">{</span><span class="nb">str</span><span class="p">(</span><span class="n">k</span><span class="p">):</span> <span class="nb">str</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
|
||||
|
||||
<span class="n">_parallel_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_ParallelConfig</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">_model_format</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_ModelFormatKind</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
<span class="n">_speculative_model</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">PrivateAttr</span><span class="p">(</span><span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
|
||||
@ -2745,6 +2853,8 @@
|
||||
<span class="nd">@field_validator</span><span class="p">(</span><span class="s2">"gpus_per_node"</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">'before'</span><span class="p">)</span>
|
||||
<span class="nd">@classmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate_gpus_per_node</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="n">info</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">getenv</span><span class="p">(</span><span class="s2">"RAY_LOCAL_WORLD_SIZE"</span><span class="p">)</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="n">info</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">"tensor_parallel_size"</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"Using default gpus_per_node: </span><span class="si">{</span><span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">device_count</span><span class="p">()</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||
@ -3366,6 +3476,15 @@
|
||||
<span class="s2">"The type of sampler to use. Options are TRTLLMSampler, TorchSampler or auto. Defaults to auto, which will use TorchSampler unless BeamSearch is requested."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"beta"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">sampler_force_async_worker</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Force usage of the async worker in the sampler for D2H "</span>
|
||||
<span class="s2">"copies, even if confidential compute is not active. Normally, the "</span>
|
||||
<span class="s2">"async worker should only be used when confidential compute is active. "</span>
|
||||
<span class="s2">"This argument is provided to enable it for testing purposes, "</span>
|
||||
<span class="s2">"irrespective of confidential compute state."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">enable_iter_perf_stats</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span><span class="s2">"Enable iteration performance statistics."</span><span class="p">,</span>
|
||||
@ -3498,6 +3617,13 @@
|
||||
<span class="s2">"Allows users to extend the functions of the RayGPUWorker class."</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">ray_placement_config</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">RayPlacementConfig</span><span class="p">]</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
<span class="s2">"Placement config for RayGPUWorker. Only used with AsyncLLM and orchestrator_type='ray'."</span><span class="p">,</span>
|
||||
<span class="n">exclude</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">status</span><span class="o">=</span><span class="s2">"prototype"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">enable_sleep</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="n">Field</span><span class="p">(</span>
|
||||
<span class="n">default</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||
<span class="n">description</span><span class="o">=</span>
|
||||
@ -3763,6 +3889,27 @@
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="TorchLlmArgs.validate_helix_tokens_per_block">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.TorchLlmArgs.validate_helix_tokens_per_block">[docs]</a>
|
||||
<span class="nd">@model_validator</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s1">'after'</span><span class="p">)</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate_helix_tokens_per_block</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'TorchLlmArgs'</span><span class="p">:</span>
|
||||
<span class="w"> </span><span class="sd">"""Validate that cp_config.tokens_per_block matches kv_cache_config.tokens_per_block when HELIX parallelism is active."""</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">context_parallel_size</span> <span class="o">==</span> <span class="mi">1</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">cp_config</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">cp_config</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="bp">self</span>
|
||||
|
||||
<span class="n">cp_type</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cp_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'cp_type'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">cp_type</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">cp_type</span><span class="p">)</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'HELIX'</span><span class="p">:</span>
|
||||
<span class="n">cp_tokens_per_block</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cp_config</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'tokens_per_block'</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="n">cp_tokens_per_block</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="n">kv_tokens_per_block</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">kv_cache_config</span><span class="o">.</span><span class="n">tokens_per_block</span>
|
||||
<span class="k">assert</span> <span class="n">cp_tokens_per_block</span> <span class="o">==</span> <span class="n">kv_tokens_per_block</span><span class="p">,</span> <span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"When HELIX parallelism is active, cp_config.tokens_per_block (</span><span class="si">{</span><span class="n">cp_tokens_per_block</span><span class="si">}</span><span class="s2">) "</span>
|
||||
<span class="sa">f</span><span class="s2">"must match kv_cache_config.tokens_per_block (</span><span class="si">{</span><span class="n">kv_tokens_per_block</span><span class="si">}</span><span class="s2">)."</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="TorchLlmArgs.warn_on_unstable_feature_usage">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.TorchLlmArgs.warn_on_unstable_feature_usage">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">warn_on_unstable_feature_usage</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'TorchLlmArgs'</span><span class="p">:</span>
|
||||
@ -3855,6 +4002,17 @@
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="TorchLlmArgs.validate_ray_placement_config">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.TorchLlmArgs.validate_ray_placement_config">[docs]</a>
|
||||
<span class="nd">@model_validator</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="s1">'after'</span><span class="p">)</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">validate_ray_placement_config</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="s1">'TorchLlmArgs'</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">ray_placement_config</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">orchestrator_type</span> <span class="o">!=</span> <span class="s2">"ray"</span><span class="p">:</span>
|
||||
<span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||
<span class="s2">"ray_placement_config is only supported with orchestrator_type='ray'"</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="bp">self</span></div>
|
||||
|
||||
|
||||
<div class="viewcode-block" id="TorchLlmArgs.get_executor_config">
|
||||
<a class="viewcode-back" href="../../../llm-api/reference.html#tensorrt_llm.llmapi.TorchLlmArgs.get_executor_config">[docs]</a>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">get_executor_config</span><span class="p">(</span>
|
||||
@ -4087,9 +4245,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -778,9 +780,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1248,9 +1250,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -886,9 +888,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1190,9 +1192,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -798,9 +800,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -815,9 +817,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1014,9 +1016,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -843,9 +845,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -674,9 +676,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -927,9 +929,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -825,9 +827,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -689,9 +691,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -815,9 +817,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -909,9 +911,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -991,9 +993,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1027,9 +1029,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1963,9 +1965,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -2870,9 +2872,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -750,9 +752,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -912,9 +914,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -840,9 +842,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1035,9 +1037,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -959,9 +961,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1062,9 +1064,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -688,9 +690,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -838,9 +840,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -780,9 +782,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -914,9 +916,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1262,9 +1264,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1107,9 +1109,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -747,9 +749,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -897,9 +899,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -2208,9 +2210,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1274,9 +1276,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -2683,9 +2685,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -812,9 +814,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -746,9 +748,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -814,9 +816,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -817,9 +819,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -859,9 +861,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -955,9 +957,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1260,9 +1262,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -947,9 +949,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1435,9 +1437,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1109,9 +1111,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1903,9 +1905,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1174,9 +1176,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -5514,9 +5516,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1118,9 +1120,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -991,6 +993,7 @@
|
||||
<span class="n">prompt_table</span><span class="p">,</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">),</span> <span class="s2">"Prompt table should be str or torch.Tensor"</span>
|
||||
<span class="n">prompt_table_data</span> <span class="o">=</span> <span class="n">prompt_table</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">dtype</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">current_stream</span><span class="p">()</span><span class="o">.</span><span class="n">synchronize</span><span class="p">()</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">prompt_table_data</span>
|
||||
|
||||
@ -1637,9 +1640,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -1850,9 +1852,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -3432,9 +3434,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -978,9 +980,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@
|
||||
<script>
|
||||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc5';
|
||||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc6';
|
||||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||||
false;
|
||||
</script>
|
||||
@ -73,7 +73,7 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<meta name="docsearch:language" content="en"/>
|
||||
<meta name="docsearch:version" content="1.2.0rc5" />
|
||||
<meta name="docsearch:version" content="1.2.0rc6" />
|
||||
|
||||
|
||||
</head>
|
||||
@ -353,6 +353,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/curl_responses_client.html">Curl Responses Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||||
@ -361,6 +362,7 @@
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||||
<li class="toctree-l2"><a class="reference internal" href="../../examples/openai_responses_client.html">OpenAI Responses Client</a></li>
|
||||
</ul>
|
||||
</details></li>
|
||||
<li class="toctree-l1"><a class="reference internal" href="../../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||||
@ -862,9 +864,13 @@
|
||||
<a class="viewcode-back" href="../../llm-api/reference.html#tensorrt_llm.llmapi.SamplingParams.params_imply_greedy_decoding">[docs]</a>
|
||||
<span class="nd">@staticmethod</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">params_imply_greedy_decoding</span><span class="p">(</span>
|
||||
<span class="o">*</span><span class="p">,</span> <span class="n">temperature</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="n">top_p</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span> <span class="n">top_k</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span>
|
||||
<span class="o">*</span><span class="p">,</span>
|
||||
<span class="n">temperature</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
|
||||
<span class="n">top_p</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">],</span>
|
||||
<span class="n">top_k</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
|
||||
<span class="n">use_beam_search</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span>
|
||||
<span class="p">):</span>
|
||||
<span class="k">return</span> <span class="p">(</span>
|
||||
<span class="k">return</span> <span class="p">(</span><span class="ow">not</span> <span class="n">use_beam_search</span><span class="p">)</span> <span class="ow">and</span> <span class="p">(</span>
|
||||
<span class="p">(</span><span class="n">temperature</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">top_p</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">top_k</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span>
|
||||
<span class="ow">or</span> <span class="n">top_k</span> <span class="o">==</span> <span class="mi">1</span>
|
||||
<span class="ow">or</span> <span class="n">top_p</span> <span class="o">==</span> <span class="mf">0.0</span>
|
||||
@ -874,10 +880,11 @@
|
||||
|
||||
<span class="nd">@property</span>
|
||||
<span class="k">def</span><span class="w"> </span><span class="nf">_greedy_decoding</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span>
|
||||
<span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beam_search</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">params_imply_greedy_decoding</span><span class="p">(</span>
|
||||
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">params_imply_greedy_decoding</span><span class="p">(</span>
|
||||
<span class="n">temperature</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">temperature</span><span class="p">,</span>
|
||||
<span class="n">top_p</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">top_p</span><span class="p">,</span>
|
||||
<span class="n">top_k</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">top_k</span><span class="p">,</span>
|
||||
<span class="n">use_beam_search</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_beam_search</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="nd">@property</span>
|
||||
@ -1192,9 +1199,9 @@
|
||||
<div class="footer-item">
|
||||
<div class="extra_footer">
|
||||
|
||||
<p>Last updated on December 07, 2025.</p>
|
||||
<p>Last updated on December 15, 2025.</p>
|
||||
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/e4c7078">e4c7078</a>.</p>
|
||||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/9ba1426">9ba1426</a>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
|
||||
@ -30,7 +30,7 @@ In this blog, we share the configurations and procedures about how to reproduce
|
||||
- [Expected Result Format](#expected-result-format-3)
|
||||
- [Exploring more ISL/OSL combinations](#exploring-more-islosl-combinations)
|
||||
- [WIP: Enable more features by default](#wip-enable-more-features-by-default)
|
||||
- [Not supported: MLA chunked context support on Hopper](#not-supported-mla-chunked-context-support-on-hopper)
|
||||
- [MLA chunked context](#mla-chunked-context)
|
||||
- [Out of memory issues](#out-of-memory-issues)
|
||||
|
||||
|
||||
@ -69,8 +69,11 @@ For NVIDIA Hopper GPUs, it's recommended to use the FP8 version of the DeepSeek
|
||||
YOUR_MODEL_PATH=<YOUR_MODEL_PATH>
|
||||
cd $YOUR_MODEL_PATH
|
||||
|
||||
## Download FP4 model for Blackwell GPUs
|
||||
git clone https://huggingface.co/nvidia/DeepSeek-R1-FP4
|
||||
## Download NVFP4 model for Blackwell GPUs
|
||||
git clone https://huggingface.co/nvidia/DeepSeek-R1-NVFP4-v2
|
||||
|
||||
## Or the 0528 version
|
||||
git clone https://huggingface.co/nvidia/DeepSeek-R1-0528-NVFP4-v2
|
||||
|
||||
## Download FP8 model for Hopper GPUs
|
||||
## FP8 model also works for Blackwell, but FP4 has the best performance on Blackwell.
|
||||
@ -248,13 +251,13 @@ To do the benchmark, run the following command:
|
||||
|
||||
```bash
|
||||
# generate synthetic dataset
|
||||
python ${YOUR_WORK_PATH}/benchmarks/cpp/prepare_dataset.py \
|
||||
--stdout \
|
||||
--tokenizer nvidia/DeepSeek-R1-FP4 \
|
||||
trtllm-bench --model nvidia/DeepSeek-R1-FP4 \
|
||||
prepare-dataset \
|
||||
--output dataset.txt \
|
||||
token-norm-dist \
|
||||
--input-mean 1024 --output-mean 2048 \
|
||||
--input-stdev 0 --output-stdev 0 \
|
||||
--num-requests 49152 > dataset.txt
|
||||
--num-requests 49152
|
||||
|
||||
YOUR_DATA_PATH=./dataset.txt
|
||||
|
||||
@ -350,13 +353,14 @@ To do the benchmark, run the following command:
|
||||
|
||||
```bash
|
||||
# generate synthetic dataset
|
||||
python ${YOUR_WORK_PATH}/benchmarks/cpp/prepare_dataset.py \
|
||||
--stdout \
|
||||
--tokenizer deepseek-ai/DeepSeek-R1 \
|
||||
trtllm-bench --model nvidia/DeepSeek-R1-FP4 \
|
||||
prepare-dataset \
|
||||
--output dataset.txt \
|
||||
token-norm-dist \
|
||||
--input-mean 1024 --output-mean 2048 \
|
||||
--input-stdev 0 --output-stdev 0 \
|
||||
--num-requests 5120 > dataset.txt
|
||||
--num-requests 5120
|
||||
|
||||
YOUR_DATA_PATH=./dataset.txt
|
||||
|
||||
cat >./extra-llm-api-config.yml<<EOF
|
||||
@ -401,10 +405,10 @@ Average request latency (ms): 181540.5739
|
||||
|
||||
## Exploring more ISL/OSL combinations
|
||||
|
||||
To benchmark TensorRT LLM on DeepSeek models with more ISL/OSL combinations, you can use `prepare_dataset.py` to generate the dataset and use similar commands mentioned in the previous section. TensorRT LLM is working on enhancements that can make the benchmark process smoother.
|
||||
To benchmark TensorRT LLM on DeepSeek models with more ISL/OSL combinations, you can use the `trtllm-bench prepare-dataset` subcommand to generate the dataset and use similar commands mentioned in the previous section. TensorRT LLM is working on enhancements that can make the benchmark process smoother.
|
||||
### WIP: Enable more features by default
|
||||
|
||||
Currently, there are some features that need to be enabled through a user-defined file `extra-llm-api-config.yml`, such as CUDA graph, overlap scheduler and attention dp. We're working on to enable those features by default, so that users can get good out-of-the-box performance on DeepSeek models.
|
||||
Currently, there are some features that need to be enabled through a user-defined file `extra-llm-api-config.yml`, such as attention dp. We're working on to enable those features by default, so that users can get good out-of-the-box performance on DeepSeek models.
|
||||
|
||||
Note that, `max_batch_size` and `max_num_tokens` can easily affect the performance. The default values for them are already carefully designed and should deliver good performance on overall cases, however, you may still need to tune it for peak performance.
|
||||
|
||||
@ -414,7 +418,7 @@ For more details on `max_batch_size` and `max_num_tokens`, refer to [Tuning Max
|
||||
|
||||
### MLA chunked context
|
||||
|
||||
MLA currently supports the chunked context feature on both Hopper and Blackwell GPUs. You can use `--enable_chunked_context` to enable it. This feature is primarily designed to reduce TPOT (Time Per Output Token). The default chunk size is set to `max_num_tokens`. If you want to achieve a lower TPOT, you can appropriately reduce the chunk size. However, please note that this will also decrease overall throughput. Therefore, a trade-off needs to be considered.
|
||||
MLA currently supports the chunked context feature on both Hopper and Blackwell GPUs. You can use `--enable_chunked_context` to enable it. This feature is primarily designed to reduce TPOT (Time Per Output Token). The default chunk size is set to `max_num_tokens`. If you want to achieve a lower TPOT, you can appropriately reduce the chunk size. However, please note that this will also decrease overall throughput. Therefore, a trade-off needs to be considered.
|
||||
|
||||
For more details on `max_num_tokens`, refer to [Tuning Max Batch Size and Max Num Tokens](../performance/performance-tuning-guide/tuning-max-batch-size-and-max-num-tokens.md).
|
||||
|
||||
|
||||
@ -46,7 +46,7 @@ In this third blog of our scaling Expert Parallelism (EP) series, we push the pe
|
||||
|
||||
The wo GEMM is the final linear layer within the multi-head attention block that produces the final outputs. While DeepSeek R1's MLA modifies the initial projections for keys and values, the wo GEMM operator remains a critical and standard component for finalizing the attention computation. In the term, "wo" is the abbreviation for the weight matrix for the output.
|
||||
|
||||
We've evaluated that quantizing the wo GEMM to FP4 still satisfies the accuracy requirements, maintaining a similar MTP accept rate (AR) while improving end-to-end performance. The [NVIDIA TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer) team has published checkpoints that additionally quantize the wo module in attention layers to FP4 on HuggingFace:
|
||||
We've evaluated that quantizing the wo GEMM to FP4 still satisfies the accuracy requirements, maintaining a similar MTP accept rate (AR) while improving end-to-end performance. The [NVIDIA Model Optimizer](https://github.com/NVIDIA/Model-Optimizer) team has published checkpoints that additionally quantize the wo module in attention layers to FP4 on HuggingFace:
|
||||
* https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2
|
||||
* https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ We have explored a mixed precision recipe, which provides a better tradeoff betw
|
||||
|
||||
*TensorRT LLM already supports [FP8 Attention](https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/models/core/deepseek_v3#fp8-kv-cache-and-mla) while for this latency scenario low-precision attention computation doesn't help with performance so we choose to use bf16 precision for the Attention Modules.
|
||||
|
||||
** nvfp4 model checkpoint is generated by the [NVIDIA TensorRT Model Optimizer toolkit](https://github.com/NVIDIA/TensorRT-Model-Optimizer).
|
||||
** nvfp4 model checkpoint is generated by the [NVIDIA Model Optimizer toolkit](https://github.com/NVIDIA/Model-Optimizer).
|
||||
|
||||
*** RouterGEMM uses bf16 inputs/weights with fp32 outputs for numerical stability
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ The mixed precision recipe for DeepSeek R1 throughput scenario is almost the sam
|
||||
* FP8 KV cache and FP8 attention, rather than BF16 precision.
|
||||
* FP4 Allgather for better communication bandwidth utilization.
|
||||
|
||||
The checkpoint used in this blog is hosted in [nvidia/DeepSeek-R1-FP4](https://huggingface.co/nvidia/DeepSeek-R1-FP4), generated by [NVIDIA Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer). The accuracy score of common dataset on this FP4 checkpoint and TensorRT LLM implementations are:
|
||||
The checkpoint used in this blog is hosted in [nvidia/DeepSeek-R1-FP4](https://huggingface.co/nvidia/DeepSeek-R1-FP4), generated by [NVIDIA Model Optimizer](https://github.com/NVIDIA/Model-Optimizer). The accuracy score of common dataset on this FP4 checkpoint and TensorRT LLM implementations are:
|
||||
|
||||
| Precision | GPQA Diamond | MATH-500
|
||||
| :-- | :-- | :-- |
|
||||
|
||||
@ -25,7 +25,7 @@ TensorRT LLM distributes the pre-built container on [NGC Catalog](https://catalo
|
||||
You can launch the container using the following command:
|
||||
|
||||
```bash
|
||||
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5
|
||||
docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc6
|
||||
```
|
||||
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ For the full syntax and argument descriptions, refer to :ref:`syntax`.
|
||||
Inference Endpoints
|
||||
-------------------
|
||||
|
||||
After you start the server, you can send inference requests through completions API and Chat API, which are compatible with corresponding OpenAI APIs. We use `TinyLlama-1.1B-Chat-v1.0 <https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0>`_ for examples in the following sections.
|
||||
After you start the server, you can send inference requests through completions API, Chat API and Responses API, which are compatible with corresponding OpenAI APIs. We use `TinyLlama-1.1B-Chat-v1.0 <https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0>`_ for examples in the following sections.
|
||||
|
||||
Chat API
|
||||
~~~~~~~~
|
||||
@ -66,6 +66,24 @@ Another example uses ``curl``:
|
||||
:language: bash
|
||||
:linenos:
|
||||
|
||||
Responses API
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
You can query Responses API with any http clients, a typical example is OpenAI Python client:
|
||||
|
||||
.. literalinclude:: ../../../../examples/serve/openai_responses_client.py
|
||||
:language: python
|
||||
:linenos:
|
||||
|
||||
Another example uses ``curl``:
|
||||
|
||||
.. literalinclude:: ../../../../examples/serve/curl_responses_client.sh
|
||||
:language: bash
|
||||
:linenos:
|
||||
|
||||
|
||||
More openai compatible examples can be found in the `compatibility examples <https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/serve/compatibility>`_ directory.
|
||||
|
||||
Multimodal Serving
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
||||
1074
latest/_sources/deployment-guide/config_table.rst.txt
Normal file
1074
latest/_sources/deployment-guide/config_table.rst.txt
Normal file
File diff suppressed because it is too large
Load Diff
@ -47,7 +47,7 @@ docker run --rm -it \
|
||||
-p 8000:8000 \
|
||||
-v ~/.cache:/root/.cache:rw \
|
||||
--name tensorrt_llm \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc6 \
|
||||
/bin/bash
|
||||
```
|
||||
|
||||
@ -66,7 +66,7 @@ We maintain YAML configuration files with recommended performance settings in th
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -74,7 +74,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/deepseek-r1-throughput.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/deepseek-r1-throughput.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
@ -90,7 +90,7 @@ To use the `DeepGEMM` MOE backend on B200/GB200, use this config instead:
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/deepseek-r1-deepgemm.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-deepgemm.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -98,7 +98,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/deepseek-r1-deepgemm.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/deepseek-r1-deepgemm.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
@ -154,7 +154,7 @@ These options provide control over TensorRT LLM's behavior and are set within th
|
||||
|
||||
#### `trust_remote_code`
|
||||
|
||||
 **Description:** Allows TensorRT LLM to download models and tokenizers from Hugging Face. This flag is passed directly to the Hugging Face API.
|
||||
* **Description:** Allows TensorRT LLM to download models and tokenizers from Hugging Face. This flag is passed directly to the Hugging Face API.
|
||||
|
||||
#### `kv_cache_config`
|
||||
|
||||
@ -429,3 +429,23 @@ $$
|
||||
$$
|
||||
\text{TPS} = \frac{\text{Num Output Tokens}}{T_{last} - T_{first}}
|
||||
$$
|
||||
|
||||
## Preconfigured Recipes
|
||||
|
||||
The following tables list recommended configurations from the comprehensive database for different performance profiles.
|
||||
|
||||
```{eval-rst}
|
||||
.. include:: note_sections.rst
|
||||
:start-after: .. start-note-traffic-patterns
|
||||
:end-before: .. end-note-traffic-patterns
|
||||
|
||||
.. include:: config_table.rst
|
||||
:start-after: .. start-deepseek-ai/DeepSeek-R1-0528
|
||||
:end-before: .. end-deepseek-ai/DeepSeek-R1-0528
|
||||
```
|
||||
|
||||
```{eval-rst}
|
||||
.. include:: config_table.rst
|
||||
:start-after: .. start-nvidia/DeepSeek-R1-0528-FP4-v2
|
||||
:end-before: .. end-nvidia/DeepSeek-R1-0528-FP4-v2
|
||||
```
|
||||
|
||||
@ -43,7 +43,7 @@ docker run --rm -it \
|
||||
-p 8000:8000 \
|
||||
-v ~/.cache:/root/.cache:rw \
|
||||
--name tensorrt_llm \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc6 \
|
||||
/bin/bash
|
||||
```
|
||||
|
||||
@ -64,7 +64,7 @@ For low-latency use cases:
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/gpt-oss-120b-latency.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-latency.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -72,7 +72,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/gpt-oss-120b-latency.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/gpt-oss-120b-latency.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
@ -88,7 +88,7 @@ For max-throughput use cases:
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/gpt-oss-120b-throughput.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-throughput.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -96,7 +96,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/gpt-oss-120b-throughput.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/gpt-oss-120b-throughput.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
@ -377,3 +377,17 @@ $$
|
||||
$$
|
||||
\text{TPS} = \frac{\text{Num Output Tokens}}{T_{last} - T_{first}}
|
||||
$$
|
||||
|
||||
## Preconfigured Recipes
|
||||
|
||||
The following table lists recommended configurations from the comprehensive database for different performance profiles.
|
||||
|
||||
```{eval-rst}
|
||||
.. include:: note_sections.rst
|
||||
:start-after: .. start-note-traffic-patterns
|
||||
:end-before: .. end-note-traffic-patterns
|
||||
|
||||
.. include:: config_table.rst
|
||||
:start-after: .. start-openai/gpt-oss-120b
|
||||
:end-before: .. end-openai/gpt-oss-120b
|
||||
```
|
||||
|
||||
@ -306,3 +306,18 @@ Run `bench.sh` to begin a serving benchmark.
|
||||
```shell
|
||||
./bench.sh
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
Since Kimi K2 Thinking has larger weight size than other models, it's possible seeing host OOM issues, as the following:
|
||||
|
||||
```log
|
||||
Loading weights: 100%|█████████████████████| 1408/1408 [03:43<00:00, 6.30it/s]
|
||||
0: [12/04/2025-18:38:28] [TRT-LLM] [RANK 0] [I] moe_load_balancer finalizing model...
|
||||
1: [nvl72136-T14:452151:0:452151] Caught signal 7 (Bus error: nonexistent physical address)
|
||||
1: ==== backtrace (tid: 452151) ====
|
||||
1: 0 /usr/local/ucx//lib/libucs.so.0(ucs_handle_error+0x2cc) [0xffff9638274c]
|
||||
1: 1 /usr/local/ucx//lib/libucs.so.0(+0x328fc) [0xffff963828fc]
|
||||
1: 2 /usr/local/ucx//lib/libucs.so.0(+0x32c78) [0xffff96382c78]
|
||||
```
|
||||
This can be addressed by mounting `tmpfs:/dev/shm:size=640G` when launching the Docker container, to increase the shm size that the container can access.
|
||||
|
||||
@ -39,7 +39,7 @@ docker run --rm -it \
|
||||
-p 8000:8000 \
|
||||
-v ~/.cache:/root/.cache:rw \
|
||||
--name tensorrt_llm \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc6 \
|
||||
/bin/bash
|
||||
```
|
||||
|
||||
@ -58,7 +58,7 @@ We maintain YAML configuration files with recommended performance settings in th
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/llama-3.3-70b.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/llama-3.3-70b.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -66,7 +66,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/llama-3.3-70b.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/llama-3.3-70b.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
|
||||
@ -38,7 +38,7 @@ docker run --rm -it \
|
||||
-p 8000:8000 \
|
||||
-v ~/.cache:/root/.cache:rw \
|
||||
--name tensorrt_llm \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \
|
||||
nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc6 \
|
||||
/bin/bash
|
||||
```
|
||||
|
||||
@ -57,7 +57,7 @@ We maintain YAML configuration files with recommended performance settings in th
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/llama-4-scout.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/llama-4-scout.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -65,7 +65,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/llama-4-scout.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/llama-4-scout.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
|
||||
@ -35,7 +35,7 @@ We maintain YAML configuration files with recommended performance settings in th
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/qwen3-next.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/qwen3-next.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -43,7 +43,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/qwen3-next.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/qwen3-next.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
|
||||
@ -40,7 +40,7 @@ We maintain YAML configuration files with recommended performance settings in th
|
||||
|
||||
```shell
|
||||
TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/qwen3.yaml
|
||||
EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml
|
||||
```
|
||||
|
||||
Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below.
|
||||
@ -48,7 +48,7 @@ Note: if you don't have access to the source code locally, you can manually crea
|
||||
````{admonition} Show code
|
||||
:class: dropdown
|
||||
|
||||
```{literalinclude} ../../../examples/configs/qwen3.yaml
|
||||
```{literalinclude} ../../../examples/configs/curated/qwen3.yaml
|
||||
---
|
||||
language: shell
|
||||
prepend: |
|
||||
|
||||
@ -6,15 +6,20 @@ Quick Start for Popular Models
|
||||
|
||||
The table below contains ``trtllm-serve`` commands that can be used to easily deploy popular models including DeepSeek-R1, gpt-oss, Llama 4, Qwen3, and more.
|
||||
|
||||
We maintain LLM API configuration files for these models containing recommended performance settings in the `examples/configs <https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/configs>`_ directory. The TensorRT LLM Docker container makes the config files available at ``/app/tensorrt_llm/examples/configs``, but you can customize this as needed:
|
||||
We maintain LLM API configuration files for these models containing recommended performance settings in two locations:
|
||||
|
||||
* **Curated Examples**: `examples/configs/curated <https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/configs/curated>`_ - Hand-picked configurations for common scenarios.
|
||||
* **Comprehensive Database**: `examples/configs/database <https://github.com/NVIDIA/TensorRT-LLM/tree/main/examples/configs/database>`_ - A more comprehensive set of known-good configurations for various GPUs and traffic patterns.
|
||||
|
||||
The TensorRT LLM Docker container makes these config files available at ``/app/tensorrt_llm/examples/configs/curated`` and ``/app/tensorrt_llm/examples/configs/database`` respectively. You can reference them as needed:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
export TRTLLM_DIR="/app/tensorrt_llm" # path to the TensorRT LLM repo in your local environment
|
||||
|
||||
.. note::
|
||||
|
||||
The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, you may benefit from additional tuning. In the future, we plan to provide more configs for a wider range of traffic patterns.
|
||||
.. include:: note_sections.rst
|
||||
:start-after: .. start-note-quick-start-isl-osl
|
||||
:end-before: .. end-note-quick-start-isl-osl
|
||||
|
||||
This table is designed to provide a straightforward starting point; for detailed model-specific deployment guides, check out the guides below.
|
||||
|
||||
@ -30,53 +35,53 @@ This table is designed to provide a straightforward starting point; for detailed
|
||||
* - `DeepSeek-R1 <https://huggingface.co/deepseek-ai/DeepSeek-R1-0528>`_
|
||||
- H100, H200
|
||||
- Max Throughput
|
||||
- `deepseek-r1-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/deepseek-r1-throughput.yaml>`_
|
||||
- ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml``
|
||||
- `deepseek-r1-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-r1-throughput.yaml>`_
|
||||
- ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml``
|
||||
* - `DeepSeek-R1 <https://huggingface.co/deepseek-ai/DeepSeek-R1-0528>`_
|
||||
- B200, GB200
|
||||
- Max Throughput
|
||||
- `deepseek-r1-deepgemm.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/deepseek-r1-deepgemm.yaml>`_
|
||||
- ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-deepgemm.yaml``
|
||||
- `deepseek-r1-deepgemm.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-r1-deepgemm.yaml>`_
|
||||
- ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-deepgemm.yaml``
|
||||
* - `DeepSeek-R1 (NVFP4) <https://huggingface.co/nvidia/DeepSeek-R1-FP4>`_
|
||||
- B200, GB200
|
||||
- Max Throughput
|
||||
- `deepseek-r1-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/deepseek-r1-throughput.yaml>`_
|
||||
- ``trtllm-serve nvidia/DeepSeek-R1-FP4 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml``
|
||||
- `deepseek-r1-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-r1-throughput.yaml>`_
|
||||
- ``trtllm-serve nvidia/DeepSeek-R1-FP4 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml``
|
||||
* - `DeepSeek-R1 (NVFP4) <https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2>`_
|
||||
- B200, GB200
|
||||
- Min Latency
|
||||
- `deepseek-r1-latency.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/deepseek-r1-latency.yaml>`_
|
||||
- ``trtllm-serve nvidia/DeepSeek-R1-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-latency.yaml``
|
||||
- `deepseek-r1-latency.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/deepseek-r1-latency.yaml>`_
|
||||
- ``trtllm-serve nvidia/DeepSeek-R1-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-latency.yaml``
|
||||
* - `gpt-oss-120b <https://huggingface.co/openai/gpt-oss-120b>`_
|
||||
- Any
|
||||
- Max Throughput
|
||||
- `gpt-oss-120b-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/gpt-oss-120b-throughput.yaml>`_
|
||||
- ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/gpt-oss-120b-throughput.yaml``
|
||||
- `gpt-oss-120b-throughput.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/gpt-oss-120b-throughput.yaml>`_
|
||||
- ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-throughput.yaml``
|
||||
* - `gpt-oss-120b <https://huggingface.co/openai/gpt-oss-120b>`_
|
||||
- Any
|
||||
- Min Latency
|
||||
- `gpt-oss-120b-latency.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/gpt-oss-120b-latency.yaml>`_
|
||||
- ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/gpt-oss-120b-latency.yaml``
|
||||
- `gpt-oss-120b-latency.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/gpt-oss-120b-latency.yaml>`_
|
||||
- ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-latency.yaml``
|
||||
* - `Qwen3-Next-80B-A3B-Thinking <https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking>`_
|
||||
- Any
|
||||
- Max Throughput
|
||||
- `qwen3-next.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/qwen3-next.yaml>`_
|
||||
- ``trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/qwen3-next.yaml``
|
||||
- `qwen3-next.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/qwen3-next.yaml>`_
|
||||
- ``trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/qwen3-next.yaml``
|
||||
* - Qwen3 family (e.g. `Qwen3-30B-A3B <https://huggingface.co/Qwen/Qwen3-30B-A3B>`_)
|
||||
- Any
|
||||
- Max Throughput
|
||||
- `qwen3.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/qwen3.yaml>`_
|
||||
- ``trtllm-serve Qwen/Qwen3-30B-A3B --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/qwen3.yaml`` (swap to another Qwen3 model name as needed)
|
||||
- `qwen3.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/qwen3.yaml>`_
|
||||
- ``trtllm-serve Qwen/Qwen3-30B-A3B --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml`` (swap to another Qwen3 model name as needed)
|
||||
* - `Llama-3.3-70B (FP8) <https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP8>`_
|
||||
- Any
|
||||
- Max Throughput
|
||||
- `llama-3.3-70b.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/llama-3.3-70b.yaml>`_
|
||||
- ``trtllm-serve nvidia/Llama-3.3-70B-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/llama-3.3-70b.yaml``
|
||||
- `llama-3.3-70b.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/llama-3.3-70b.yaml>`_
|
||||
- ``trtllm-serve nvidia/Llama-3.3-70B-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/llama-3.3-70b.yaml``
|
||||
* - `Llama 4 Scout (FP8) <https://huggingface.co/nvidia/Llama-4-Scout-17B-16E-Instruct-FP8>`_
|
||||
- Any
|
||||
- Max Throughput
|
||||
- `llama-4-scout.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/llama-4-scout.yaml>`_
|
||||
- ``trtllm-serve nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/llama-4-scout.yaml``
|
||||
- `llama-4-scout.yaml <https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/configs/curated/llama-4-scout.yaml>`_
|
||||
- ``trtllm-serve nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/llama-4-scout.yaml``
|
||||
|
||||
Model-Specific Deployment Guides
|
||||
---------------------------------
|
||||
@ -94,3 +99,10 @@ The deployment guides below provide more detailed instructions for serving speci
|
||||
deployment-guide-for-qwen3-on-trtllm.md
|
||||
deployment-guide-for-qwen3-next-on-trtllm.md
|
||||
deployment-guide-for-kimi-k2-thinking-on-trtllm.md
|
||||
|
||||
Comprehensive Configuration Database
|
||||
------------------------------------
|
||||
|
||||
The table below lists all available pre-configured model scenarios in the TensorRT LLM configuration database. Each row represents a specific model, GPU, and performance profile combination with recommended request settings.
|
||||
|
||||
.. include:: config_table.rst
|
||||
|
||||
36
latest/_sources/deployment-guide/note_sections.rst.txt
Normal file
36
latest/_sources/deployment-guide/note_sections.rst.txt
Normal file
@ -0,0 +1,36 @@
|
||||
..
|
||||
Reusable note sections for deployment guides.
|
||||
Include specific notes using:
|
||||
|
||||
.. include:: note_sections.rst
|
||||
:start-after: .. start-note-<name>
|
||||
:end-before: .. end-note-<name>
|
||||
|
||||
.. start-note-traffic-patterns
|
||||
|
||||
.. note::
|
||||
|
||||
**Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length)
|
||||
values in each configuration represent the **maximum supported values** for that config.
|
||||
Requests exceeding these limits may result in errors.
|
||||
|
||||
To handle requests with input sequences **longer than the configured ISL**, add the following
|
||||
to your config file:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
enable_chunked_prefill: true
|
||||
|
||||
This enables chunked prefill, which processes long input sequences in chunks rather than
|
||||
requiring them to fit within a single prefill operation. Note that enabling chunked prefill
|
||||
does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL.
|
||||
|
||||
.. end-note-traffic-patterns
|
||||
|
||||
.. start-note-quick-start-isl-osl
|
||||
|
||||
.. note::
|
||||
|
||||
The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, refer to the :ref:`Comprehensive Configuration Database` section below which covers a larger set of traffic patterns and performance profiles.
|
||||
|
||||
.. end-note-quick-start-isl-osl
|
||||
@ -72,10 +72,12 @@ Say we want to profile iterations 100 to 150 on a `trtllm-bench`/`trtllm-serve`
|
||||
#!/bin/bash
|
||||
|
||||
# Prepare dataset for the benchmark
|
||||
python3 benchmarks/cpp/prepare_dataset.py \
|
||||
--tokenizer=${MODEL_PATH} \
|
||||
--stdout token-norm-dist --num-requests=${NUM_SAMPLES} \
|
||||
--input-mean=1000 --output-mean=1000 --input-stdev=0 --output-stdev=0 > /tmp/dataset.txt
|
||||
trtllm-bench --model ${MODEL_PATH} \
|
||||
prepare-dataset \
|
||||
--output dataset.txt \
|
||||
token-norm-dist \
|
||||
--num-requests=${NUM_SAMPLES} \
|
||||
--input-mean=1000 --output-mean=1000 --input-stdev=0 --output-stdev=0
|
||||
|
||||
# Benchmark and profile
|
||||
TLLM_PROFILE_START_STOP=100-150 nsys profile \
|
||||
|
||||
@ -152,7 +152,7 @@ directory. For example, to generate a synthetic dataset of 1000 requests with a
|
||||
128/128 for [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B), run:
|
||||
|
||||
```shell
|
||||
python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer meta-llama/Llama-3.1-8B token-norm-dist --input-mean 128 --output-mean 128 --input-stdev 0 --output-stdev 0 --num-requests 1000 > /tmp/synthetic_128_128.txt
|
||||
trtllm-bench --model meta-llama/Llama-3.1-8B prepare-dataset --output /tmp/synthetic_128_128.txt token-norm-dist --input-mean 128 --output-mean 128 --input-stdev 0 --output-stdev 0 --num-requests 1000
|
||||
```
|
||||
|
||||
### Running with the PyTorch Workflow
|
||||
@ -233,13 +233,13 @@ The PyTorch workflow supports benchmarking with LoRA (Low-Rank Adaptation) adapt
|
||||
|
||||
**Preparing LoRA Dataset**
|
||||
|
||||
Use `prepare_dataset.py` with LoRA-specific options to generate requests with LoRA metadata:
|
||||
Use `trtllm-bench prepare-dataset` with LoRA-specific options to generate requests with LoRA metadata:
|
||||
|
||||
```shell
|
||||
python3 benchmarks/cpp/prepare_dataset.py \
|
||||
--stdout \
|
||||
trtllm-bench \
|
||||
--model /path/to/tokenizer \
|
||||
prepare-dataset \
|
||||
--rand-task-id 0 1 \
|
||||
--tokenizer /path/to/tokenizer \
|
||||
--lora-dir /path/to/loras \
|
||||
token-norm-dist \
|
||||
--num-requests 100 \
|
||||
@ -310,17 +310,18 @@ Each subdirectory should contain the LoRA adapter files for that specific task.
|
||||
To benchmark multi-modal models with PyTorch workflow, you can follow the similar approach as above.
|
||||
|
||||
First, prepare the dataset:
|
||||
```python
|
||||
python ./benchmarks/cpp/prepare_dataset.py \
|
||||
--tokenizer Qwen/Qwen2-VL-2B-Instruct \
|
||||
--stdout \
|
||||
dataset \
|
||||
```bash
|
||||
trtllm-bench \
|
||||
--model Qwen/Qwen2-VL-2B-Instruct \
|
||||
prepare-dataset \
|
||||
--output mm_data.jsonl
|
||||
real-dataset
|
||||
--dataset-name lmms-lab/MMMU \
|
||||
--dataset-split test \
|
||||
--dataset-image-key image \
|
||||
--dataset-prompt-key question \
|
||||
--num-requests 10 \
|
||||
--output-len-dist 128,5 > mm_data.jsonl
|
||||
--output-len-dist 128,5
|
||||
```
|
||||
It will download the media files to `/tmp` directory and prepare the dataset with their paths. Note that the `prompt` fields are texts and not tokenized ids. This is due to the fact that
|
||||
the `prompt` and the media (image/video) are processed by a preprocessor for multimodal files.
|
||||
@ -423,10 +424,10 @@ checkpoint. For the Llama-3.1 models, TensorRT LLM provides the following checkp
|
||||
- [`nvidia/Llama-3.1-70B-Instruct-FP8`](https://huggingface.co/nvidia/Llama-3.1-70B-Instruct-FP8)
|
||||
- [`nvidia/Llama-3.1-405B-Instruct-FP8`](https://huggingface.co/nvidia/Llama-3.1-405B-Instruct-FP8)
|
||||
|
||||
To understand more about how to quantize your own checkpoints, refer to ModelOpt [documentation](https://nvidia.github.io/TensorRT-Model-Optimizer/deployment/1_tensorrt_llm.html).
|
||||
To understand more about how to quantize your own checkpoints, refer to ModelOpt [documentation](https://nvidia.github.io/Model-Optimizer/deployment/1_tensorrt_llm.html).
|
||||
|
||||
`trtllm-bench` utilizes the `hf_quant_config.json` file present in the pre-quantized checkpoints above. The configuration
|
||||
file is present in checkpoints quantized with [TensorRT Model Optimizer](https://github.com/NVIDIA/TensorRT-Model-Optimizer)
|
||||
file is present in checkpoints quantized with [Model Optimizer](https://github.com/NVIDIA/Model-Optimizer)
|
||||
and describes the compute and KV cache quantization that checkpoint was compiled with. For example, from the checkpoints
|
||||
above:
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ and shows the throughput scenario under maximum load. The reported metric is `To
|
||||
|
||||
The performance numbers below were collected using the steps described in this document.
|
||||
|
||||
Testing was performed on models with weights quantized using [ModelOpt](https://nvidia.github.io/TensorRT-Model-Optimizer/#) and published by NVIDIA on the [Model Optimizer HuggingFace Collection](https://huggingface.co/collections/nvidia/model-optimizer-66aa84f7966b3150262481a4).
|
||||
Testing was performed on models with weights quantized using [ModelOpt](https://nvidia.github.io/Model-Optimizer/#) and published by NVIDIA on the [Model Optimizer HuggingFace Collection](https://huggingface.co/collections/nvidia/model-optimizer-66aa84f7966b3150262481a4).
|
||||
|
||||
*(NEW for v1.0) RTX 6000 Pro Blackwell Server Edition Benchmarks:*
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ Curl Chat Client
|
||||
================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/e4c707845ff58fcc0b1d87afb4dd0e64885c780a/examples/serve/curl_chat_client.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/9ba14263db0045ed3fa0860f949b5ce320107eb3/examples/serve/curl_chat_client.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/curl_chat_client.sh
|
||||
:lines: 1-11
|
||||
|
||||
@ -2,7 +2,7 @@ Curl Chat Client For Multimodal
|
||||
===============================
|
||||
Refer to the `trtllm-serve documentation <https://nvidia.github.io/TensorRT-LLM/commands/trtllm-serve.html>`_ for starting a server.
|
||||
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/e4c707845ff58fcc0b1d87afb4dd0e64885c780a/examples/serve/curl_chat_client_for_multimodal.sh.
|
||||
Source https://github.com/NVIDIA/TensorRT-LLM/blob/9ba14263db0045ed3fa0860f949b5ce320107eb3/examples/serve/curl_chat_client_for_multimodal.sh.
|
||||
|
||||
.. literalinclude:: ../../../examples/serve/curl_chat_client_for_multimodal.sh
|
||||
:lines: 1-88
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user