mirror of
https://github.com/NVIDIA/TensorRT-LLM.git
synced 2026-01-14 06:27:45 +08:00
14001 lines
2.7 MiB
14001 lines
2.7 MiB
|
||
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="../" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Executor — TensorRT LLM</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
<!--
|
||
this give us a css class that will be invisible only if js is disabled
|
||
-->
|
||
<noscript>
|
||
<style>
|
||
.pst-js-only { display: none !important; }
|
||
|
||
</style>
|
||
</noscript>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
|
||
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/styles/nvidia-sphinx-theme.css?v=933278ad" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/autodoc_pydantic.css" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/custom.css?v=19d20f17" />
|
||
|
||
<!-- So that users can add custom icons -->
|
||
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
|
||
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
|
||
|
||
|
||
|
||
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
|
||
<script src="../_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="../_static/copybutton.js?v=65e89d2a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="../_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = '_cpp_gen/executor';</script>
|
||
<script>
|
||
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
|
||
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
|
||
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '1.2.0rc4';
|
||
DOCUMENTATION_OPTIONS.show_version_warning_banner =
|
||
false;
|
||
</script>
|
||
|
||
<link rel="icon" href="../_static/favicon.png"/>
|
||
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
<meta name="docsearch:version" content="1.2.0rc4" />
|
||
|
||
|
||
</head>
|
||
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<dialog id="pst-search-dialog">
|
||
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="../search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
placeholder="Search the docs ..."
|
||
aria-label="Search the docs ..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form>
|
||
</dialog>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
<div class="bd-header__inner bd-page-width">
|
||
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button>
|
||
|
||
|
||
<div class="col-lg-3 navbar-header-items__start">
|
||
|
||
<div class="navbar-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../index.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT LLM - Home"/>
|
||
<img src="../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT LLM - Home"/>
|
||
|
||
|
||
<p class="title logo__title">TensorRT LLM</p>
|
||
|
||
</a></div>
|
||
|
||
</div>
|
||
|
||
<div class="col-lg-9 navbar-header-items">
|
||
|
||
<div class="me-auto navbar-header-items__center">
|
||
|
||
<div class="navbar-item">
|
||
|
||
|
||
<div class="version-switcher__container dropdown pst-js-only">
|
||
<button id="pst-version-switcher-button-2"
|
||
type="button"
|
||
class="version-switcher__button btn btn-sm dropdown-toggle"
|
||
data-bs-toggle="dropdown"
|
||
aria-haspopup="listbox"
|
||
aria-controls="pst-version-switcher-list-2"
|
||
aria-label="Version switcher list"
|
||
>
|
||
Choose version <!-- this text may get changed later by javascript -->
|
||
<span class="caret"></span>
|
||
</button>
|
||
<div id="pst-version-switcher-list-2"
|
||
class="version-switcher__menu dropdown-menu list-group-flush py-0"
|
||
role="listbox" aria-labelledby="pst-version-switcher-button-2">
|
||
<!-- dropdown will be populated by javascript on page load -->
|
||
</div>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="navbar-header-items__end">
|
||
|
||
<div class="navbar-item navbar-persistent--container">
|
||
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
</div>
|
||
|
||
|
||
<div class="navbar-item">
|
||
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="navbar-persistent--mobile">
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
</div>
|
||
|
||
|
||
|
||
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
|
||
<span class="fa-solid fa-outdent"></span>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<dialog id="pst-primary-sidebar-modal"></dialog>
|
||
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="../index.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT LLM - Home"/>
|
||
<img src="../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT LLM - Home"/>
|
||
|
||
|
||
<p class="title logo__title">TensorRT LLM</p>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
<div class="sidebar-header-items__center">
|
||
|
||
|
||
|
||
<div class="navbar-item">
|
||
|
||
|
||
<div class="version-switcher__container dropdown pst-js-only">
|
||
<button id="pst-version-switcher-button-3"
|
||
type="button"
|
||
class="version-switcher__button btn btn-sm dropdown-toggle"
|
||
data-bs-toggle="dropdown"
|
||
aria-haspopup="listbox"
|
||
aria-controls="pst-version-switcher-list-3"
|
||
aria-label="Version switcher list"
|
||
>
|
||
Choose version <!-- this text may get changed later by javascript -->
|
||
<span class="caret"></span>
|
||
</button>
|
||
<div id="pst-version-switcher-list-3"
|
||
class="version-switcher__menu dropdown-menu list-group-flush py-0"
|
||
role="listbox" aria-labelledby="pst-version-switcher-button-3">
|
||
<!-- dropdown will be populated by javascript on page load -->
|
||
</div>
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="sidebar-header-items__end">
|
||
|
||
<div class="navbar-item">
|
||
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
<nav class="bd-docs-nav bd-links"
|
||
aria-label="Table of Contents">
|
||
<p class="bd-links__title" role="heading" aria-level="1">Table of Contents</p>
|
||
<div class="bd-toc-item navbar-nav"><p aria-level="2" class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../overview.html">Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../quick-start-guide.html">Quick Start Guide</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../installation/index.html">Installation</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../installation/containers.html">Pre-built release container images on NGC</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../installation/linux.html">Installing on Linux via <code class="docutils literal notranslate"><span class="pre">pip</span></code></a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../installation/build-from-source-linux.html">Building from Source Code on Linux</a></li>
|
||
</ul>
|
||
</details></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deployment Guide</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/llm_api_examples.html">LLM Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference.html">Generate text</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async.html">Generate text asynchronously</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async_streaming.html">Generate text in streaming</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_distributed.html">Distributed LLM Generation</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_sparse_attention.html">Sparse Attention</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_speculative_decoding.html">Speculative Decoding</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_connector.html">KV Cache Connector</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_kv_cache_offloading.html">KV Cache Offloading</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_runtime.html">Runtime Configuration Examples</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_sampling.html">Sampling Techniques Showcase</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_llm_distributed.html">Run LLM-API with pytorch backend on Slurm</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_bench.html">Run trtllm-bench with pytorch backend on Slurm</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_serve.html">Run trtllm-serve with pytorch backend on Slurm</a></li>
|
||
</ul>
|
||
</details></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/trtllm_serve_examples.html">Online Serving Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client.html">Curl Chat Client</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_completion_client.html">Curl Completion Client</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client.html">Genai Perf Client</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_chat_client.html">OpenAI Chat Client</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_chat_client_for_multimodal.html">OpenAI Chat Client for Multimodal</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_for_lora.html">Openai Completion Client For Lora</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client_json_schema.html">OpenAI Completion Client with JSON Schema</a></li>
|
||
</ul>
|
||
</details></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../examples/dynamo_k8s_example.html">Dynamo K8s Example</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../deployment-guide/index.html">Model Recipes</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.html">Deployment Guide for DeepSeek R1 on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.html">Deployment Guide for Llama3.3 70B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.html">Deployment Guide for Llama4 Scout 17B on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.html">Deployment Guide for GPT-OSS on TensorRT-LLM - Blackwell Hardware</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.html">Deployment Guide for Qwen3 Next on TensorRT LLM - Blackwell & Hopper Hardware</a></li>
|
||
</ul>
|
||
</details></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Models</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../models/supported-models.html">Supported Models</a></li>
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="../models/adding-new-model.html">Adding a New Model</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">CLI Reference</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../commands/trtllm-bench.html">trtllm-bench</a></li>
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="../commands/trtllm-eval.html">trtllm-eval</a></li>
|
||
<li class="toctree-l1 has-children"><a class="reference internal" href="../commands/trtllm-serve/index.html">trtllm-serve</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="../commands/trtllm-serve/trtllm-serve.html">trtllm-serve</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="../commands/trtllm-serve/run-benchmark-with-trtllm-serve.html">Run benchmarking with <code class="docutils literal notranslate"><span class="pre">trtllm-serve</span></code></a></li>
|
||
</ul>
|
||
</details></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">API Reference</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../llm-api/index.html">LLM API Introduction</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../llm-api/reference.html">API Reference</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Features</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/feature-combination-matrix.html">Feature Combination Matrix</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/attention.html">Multi-Head, Multi-Query, and Group-Query Attention</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/disagg-serving.html">Disaggregated Serving</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/kvcache.html">KV Cache System</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/long-sequence.html">Long Sequences</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/lora.html">LoRA (Low-Rank Adaptation)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/multi-modality.html">Multimodal Support in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/overlap-scheduler.html">Overlap Scheduler</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/paged-attention-ifb-scheduler.html">Paged Attention, IFB, and Request Scheduling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/parallel-strategy.html">Parallelism in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/quantization.html">Quantization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/sampling.html">Sampling</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/additional-outputs.html">Additional Outputs</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/speculative-decoding.html">Speculative Decoding</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/checkpoint-loading.html">Checkpoint Loading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/auto_deploy/auto-deploy.html">AutoDeploy (Prototype)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/ray-orchestrator.html">Ray Orchestrator (Prototype)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../features/torch_compile_and_piecewise_cuda_graph.html">Torch Compile & Piecewise CUDA Graph</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Developer Guide</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/overview.html">Architecture Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/perf-analysis.html">Performance Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/perf-benchmarking.html">TensorRT LLM Benchmarking</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/ci-overview.html">Continuous Integration Overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/dev-containers.html">Using Dev Containers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/api-change.html">LLM API Change Guide</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../developer-guide/kv-transfer.html">Introduction to KV Cache Transmission</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Blogs</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog10_ADP_Balance_Strategy.html">ADP Balance Strategy</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog11_GPT_OSS_Eagle3.html">Running GPT-OSS-120B with Eagle3 Speculative Decoding on GB200/B200 (TensorRT LLM)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog12_Combining_Guided_Decoding_and_Speculative_Decoding.html">Combining Guided Decoding and Speculative Decoding: Making CPU and GPU Cooperate Seamlessly</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.html">Inference Time Compute Implementation in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog14_Scaling_Expert_Parallelism_in_TensorRT-LLM_part3.html">Scaling Expert Parallelism in TensorRT LLM (Part 3: Pushing the Performance Boundary)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog2_DeepSeek_R1_MTP_Implementation_and_Optimization.html">DeepSeek R1 MTP Implementation and Optimization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog3_Optimizing_DeepSeek_R1_Throughput_on_NVIDIA_Blackwell_GPUs.html">Optimizing DeepSeek R1 Throughput on NVIDIA Blackwell GPUs: A Deep Dive for Developers</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog4_Scaling_Expert_Parallelism_in_TensorRT-LLM.html">Scaling Expert Parallelism in TensorRT LLM (Part 1: Design and Implementation of Large-scale EP)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog5_Disaggregated_Serving_in_TensorRT-LLM.html">Disaggregated Serving in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog6_Llama4_maverick_eagle_guide.html">How to launch Llama4 Maverick + Eagle3 TensorRT LLM server</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog7_NGram_performance_Analysis_And_Auto_Enablement.html">N-Gram Speculative Decoding in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog8_Scaling_Expert_Parallelism_in_TensorRT-LLM_part2.html">Scaling Expert Parallelism in TensorRT LLM (Part 2: Performance Status and Optimization)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog9_Deploying_GPT_OSS_on_TRTLLM.html">Running a High Performance GPT-OSS-120B Inference Server with TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/Best_perf_practice_on_DeepSeek-R1_in_TensorRT-LLM.html">How to get best performance on DeepSeek-R1 in TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/H200launch.html">H200 achieves nearly 12,000 tokens/sec on Llama2-13B with TensorRT LLM</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/XQA-kernel.html">New XQA-kernel provides 2.4x more Llama-70B throughput within the same latency budget</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../blogs/H100vsA100.html">H100 has 4.6x A100 Performance in TensorRT LLM, achieving 10,000 tok/s at 100ms to first token</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Quick Links</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM/releases">Releases</a></li>
|
||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM">Github Code</a></li>
|
||
<li class="toctree-l1"><a class="reference external" href="https://github.com/NVIDIA/TensorRT-LLM/issues?q=is%3Aissue%20state%3Aopen%20label%3Aroadmap">Roadmap</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Use TensorRT Engine</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="../legacy/tensorrt_quickstart.html">LLM API with TensorRT Engine</a></li>
|
||
</ul>
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<nav aria-label="Breadcrumb" class="d-print-none">
|
||
<ul class="bd-breadcrumbs">
|
||
|
||
<li class="breadcrumb-item breadcrumb-home">
|
||
<a href="../index.html" class="nav-link" aria-label="Home">
|
||
<i class="fa-solid fa-home"></i>
|
||
</a>
|
||
</li>
|
||
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Executor</span></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section id="executor">
|
||
<h1>Executor<a class="headerlink" href="#executor" title="Link to this heading">#</a></h1>
|
||
<section id="disaggserverutil-h">
|
||
<h2>disaggServerUtil.h<a class="headerlink" href="#disaggserverutil-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv412tensorrt_llm">
|
||
<span id="_CPPv312tensorrt_llm"></span><span id="_CPPv212tensorrt_llm"></span><span id="tensorrt_llm"></span><span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><a class="headerlink" href="#_CPPv412tensorrt_llm" title="Link to this definition">#</a><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executorE"></span><span id="_CPPv2N12tensorrt_llm8executorE"></span><span id="tensorrt_llm::executor"></span><span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executorE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executorE"></span><span id="tensorrt_llm::executor::disagg_executor"></span><span class="target" id="namespacetensorrt__llm_1_1executor_1_1disagg__executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disagg_executor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorE"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DisaggExecutorOrchestrator</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator26DisaggExecutorOrchestratorERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorIN8executor14ExecutorConfigEEERKNSt6vectorIN8executor14ExecutorConfigEEEbb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator26DisaggExecutorOrchestratorERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorIN8executor14ExecutorConfigEEERKNSt6vectorIN8executor14ExecutorConfigEEEbb"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator26DisaggExecutorOrchestratorERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorIN8executor14ExecutorConfigEEERKNSt6vectorIN8executor14ExecutorConfigEEEbb"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::DisaggExecutorOrchestrator__std::vector:std::filesystem::path:CR.std::vector:std::filesystem::path:CR.std::vector:executor::ExecutorConfig:CR.std::vector:executor::ExecutorConfig:CR.b.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a805b4f7a36690f40856f115ff5fa3d86"></span><span class="sig-name descname"><span class="n"><span class="pre">DisaggExecutorOrchestrator</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">ctxEnginePaths</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">genEnginePaths</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">ctxExecutorConfigs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">genExecutorConfigs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hasContextAwaitThreads</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hasGenAwaitThreads</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator26DisaggExecutorOrchestratorERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorIN8executor14ExecutorConfigEEERKNSt6vectorIN8executor14ExecutorConfigEEEbb" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Constructs a <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator"><span class="std std-ref">DisaggExecutorOrchestrator</span></a> object. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>ctxEnginePaths</strong> – A vector of file paths to context engine files. </p></li>
|
||
<li><p><strong>genEnginePaths</strong> – A vector of file paths to generation engine files. </p></li>
|
||
<li><p><strong>ctxExecutorConfigs</strong> – A vector of <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1ExecutorConfig"><span class="std std-ref">ExecutorConfig</span></a> for context executors. </p></li>
|
||
<li><p><strong>genExecutorConfigs</strong> – A vector of <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1ExecutorConfig"><span class="std std-ref">ExecutorConfig</span></a> for generation executors. </p></li>
|
||
<li><p><strong>hasContextAwaitThreads</strong> – Whether or not there are threads that receive response for each generation executor. </p></li>
|
||
<li><p><strong>hasGenAwaitThreads</strong> – Whether or not there are threads that receive response for each generation executor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator14enqueueContextERKNSt6vectorIN5texec7RequestEEENSt8optionalIiEEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator14enqueueContextERKNSt6vectorIN5texec7RequestEEENSt8optionalIiEEb"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator14enqueueContextERKNSt6vectorIN5texec7RequestEEENSt8optionalIiEEb"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::enqueueContext__std::vector:texec::Request:CR.std::optional:i:.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a070294bc1a93c30ef3545760a96610e1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enqueueContext</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">texec</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Request</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requests</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">selectContextId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batch</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator14enqueueContextERKNSt6vectorIN5texec7RequestEEENSt8optionalIiEEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enqueue context-only requests to context executors. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>requests</strong> – A vector of context-only requests. </p></li>
|
||
<li><p><strong>selectContextId</strong> – The index of the context executor to use. If <code class="docutils literal notranslate"><span class="pre">std::nullopt</span></code>, the executor that has the smallest number of inflight requests will be used. </p></li>
|
||
<li><p><strong>batch</strong> – If true,enqueue requests in same context executor.If false, will try to use a different executor for each request. </p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of global request ids, corresponding to the order of the requests in <code class="docutils literal notranslate"><span class="pre">requests</span></code>, the id returned may be different from the request id in each executor. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator17enqueueGenerationERKNSt6vectorIN5texec7RequestEEERKNSt6vectorI6IdTypeEENSt8optionalIiEEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator17enqueueGenerationERKNSt6vectorIN5texec7RequestEEERKNSt6vectorI6IdTypeEENSt8optionalIiEEb"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator17enqueueGenerationERKNSt6vectorIN5texec7RequestEEERKNSt6vectorI6IdTypeEENSt8optionalIiEEb"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::enqueueGeneration__std::vector:texec::Request:CR.std::vector:IdType:CR.std::optional:i:.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1ae21ddfaf38813eb8a9d50ee0a6d81344"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enqueueGeneration</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">texec</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Request</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requests</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">globalRequestIds</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">selectGenIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batch</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator17enqueueGenerationERKNSt6vectorIN5texec7RequestEEERKNSt6vectorI6IdTypeEENSt8optionalIiEEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enqueue generation-only requests to generation executors. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>requests</strong> – A vector of generation-only requests. </p></li>
|
||
<li><p><strong>globalRequestIds</strong> – A vector of global request ids, corresponding to the order of the requests,and must be the ids returned by the enqueueContext function. </p></li>
|
||
<li><p><strong>selectGenIdx</strong> – The index of the generation executor to use. If <code class="docutils literal notranslate"><span class="pre">std::nullopt</span></code>, the executor that has the smallest number of inflight requests will be used. </p></li>
|
||
<li><p><strong>batch</strong> – If true,enqueue requests in same generation executor.If false, will try to use a different executor for each request. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator21awaitContextResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator21awaitContextResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator21awaitContextResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::awaitContextResponses__std::optional:std::chrono::milliseconds:CR.std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1add36b49f7e4ba267bf310413cb56b455"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">awaitContextResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">timeout</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">contextIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator21awaitContextResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Await for context responses. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>timeout</strong> – The maximum time to wait for new responses </p></li>
|
||
<li><p><strong>contextIdx</strong> – The index of the context executor to use. If <code class="docutils literal notranslate"><span class="pre">std::nullopt</span></code>, return ready responses in all context executors,if <code class="docutils literal notranslate"><span class="pre">hasContextAwaitThreads</span></code> is true, then this parameter must be std::nullopt. </p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of responses with corresponding global request ids </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator24awaitGenerationResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator24awaitGenerationResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator24awaitGenerationResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::awaitGenerationResponses__std::optional:std::chrono::milliseconds:CR.std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a19bae735cb389e30f8baf0141a547b0a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">awaitGenerationResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">timeout</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">genIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator24awaitGenerationResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Await for generation responses. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>timeout</strong> – The maximum time to wait for new responses. </p></li>
|
||
<li><p><strong>genIdx</strong> – The index of the generation executor to use. If <code class="docutils literal notranslate"><span class="pre">std::nullopt</span></code>, return ready responses in all generation executors,if <code class="docutils literal notranslate"><span class="pre">hasGenAwaitThreads</span></code> is true, then this parameter must be std::nullopt. </p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of responses with corresponding global request ids. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator10canEnqueueEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator10canEnqueueEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator10canEnqueueEv"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::canEnqueueC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a63fdb48970256462e3180d018097ad92"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">canEnqueue</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator10canEnqueueEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if the current process is allowed to enqueueRequests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator19getContextExecutorsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator19getContextExecutorsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator19getContextExecutorsEv"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::getContextExecutorsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a6385de81f6584a23cfe8f1584ab206db"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">texec</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Executor</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getContextExecutors</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator19getContextExecutorsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get context executors. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator15getGenExecutorsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator15getGenExecutorsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator15getGenExecutorsEv"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::getGenExecutorsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1a6d203b7d545eb0a5b7d0ff9f972205bd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">texec</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Executor</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getGenExecutors</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator15getGenExecutorsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get generation executors. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorD0Ev"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::~DisaggExecutorOrchestrator"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1ae0b67aacedf99304a579568cdc3141f6"></span><span class="sig-name descname"><span class="n"><span class="pre">~DisaggExecutorOrchestrator</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator5mImplE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator5mImplE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator5mImplE"></span><span id="tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator::mImpl__std::unique_ptr:Impl:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1disagg__executor_1_1DisaggExecutorOrchestrator_1ac7f0969ddf0b6fef4c495414095aa6aa"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Impl</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mImpl</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator5mImplE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ResponseWithId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERRN12tensorrt_llm8executor8ResponseE6IdType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERRN12tensorrt_llm8executor8ResponseE6IdType"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERRN12tensorrt_llm8executor8ResponseE6IdType"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId__tensorrt_llm::executor::ResponseRR.IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a24568b6374b964b011a37252a872b480"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ResponseWithId</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">response</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gid</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERRN12tensorrt_llm8executor8ResponseE6IdType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERKN12tensorrt_llm8executor8ResponseE6IdType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERKN12tensorrt_llm8executor8ResponseE6IdType"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERKN12tensorrt_llm8executor8ResponseE6IdType"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId__tensorrt_llm::executor::ResponseCR.IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a9b9497c2aa41a5d790d8ff7f385f10f1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ResponseWithId</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">response</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gid</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERKN12tensorrt_llm8executor8ResponseE6IdType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId__ResponseWithIdRR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a67bfcc9b54f71b7159c39e217f7dece8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ResponseWithId</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId" title="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId__ResponseWithIdCR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a0b15ba9e365a1bfd3f2a2a5c0c8d18dd"></span><span class="sig-name descname"><span class="n"><span class="pre">ResponseWithId</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId" title="tensorrt_llm::executor::disagg_executor::ResponseWithId::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERR14ResponseWithId">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERR14ResponseWithId"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERR14ResponseWithId"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::assign-operator__ResponseWithIdRR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a69d5eab7c0ed997cc298c0a39f6844ec"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERR14ResponseWithId" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERK14ResponseWithId">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERK14ResponseWithId"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERK14ResponseWithId"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::assign-operator__ResponseWithIdCR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a187091dc435f65eaaf5a062429c1ce80"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE" title="tensorrt_llm::executor::disagg_executor::ResponseWithId"><span class="n"><span class="pre">ResponseWithId</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERK14ResponseWithId" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithIdD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithIdD0Ev"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::~ResponseWithId"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a5042e819fe68dcf22e34869748154e3a"></span><span class="sig-name descname"><span class="n"><span class="pre">~ResponseWithId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId8responseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId8responseE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId8responseE"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::response__tensorrt_llm::executor::Response"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a61c708fecf2bf73f3e98b6b3c8b7e558"></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">response</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId8responseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId3gidE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15disagg_executor14ResponseWithId3gidE"></span><span id="_CPPv2N12tensorrt_llm8executor15disagg_executor14ResponseWithId3gidE"></span><span id="tensorrt_llm::executor::disagg_executor::ResponseWithId::gid__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1disagg__executor_1_1ResponseWithId_1a39d756b6d5a76709a6fb505561a33c78"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gid</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId3gidE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="tensor-h">
|
||
<h2>tensor.h<a class="headerlink" href="#tensor-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5ShapeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5ShapeE"></span><span id="_CPPv2N12tensorrt_llm8executor5ShapeE"></span><span id="tensorrt_llm::executor::Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Shape</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ArrayView</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6detailE" title="tensorrt_llm::executor::detail"><span class="n"><span class="pre">detail</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6detail9DimType64E" title="tensorrt_llm::executor::detail::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5Shape4BaseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5Shape4BaseE"></span><span id="_CPPv2N12tensorrt_llm8executor5Shape4BaseE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape_1abea4be68b98825ce00bd278dd61492ac"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Base</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ArrayView</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6detailE" title="tensorrt_llm::executor::detail"><span class="n"><span class="pre">detail</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6detail9DimType64E" title="tensorrt_llm::executor::detail::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5Shape4BaseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5Shape9DimType64E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5Shape9DimType64E"></span><span id="_CPPv2N12tensorrt_llm8executor5Shape9DimType64E"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape_1a83c46d18b03c349f865a290209be5528"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DimType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_cv_t</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5Shape4BaseE" title="tensorrt_llm::executor::Shape::Base"><span class="n"><span class="pre">Base</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">value_type</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5Shape9DimType64E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5Shape5ShapeEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5Shape5ShapeEv"></span><span id="_CPPv2N12tensorrt_llm8executor5Shape5ShapeEv"></span><span id="tensorrt_llm::executor::Shape::Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape_1a476b2b6a441e9fcbb8cad03b354b6681"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Shape</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5Shape5ShapeEPK9DimType64N4Base9size_typeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5Shape5ShapeEPK9DimType64N4Base9size_typeE"></span><span id="_CPPv2N12tensorrt_llm8executor5Shape5ShapeEPK9DimType64N4Base9size_typeE"></span><span id="tensorrt_llm::executor::Shape::Shape__DimType64CP.Base::size_type"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape_1aadbc8651b94baa75a3e2eb2cff889293"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Shape</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5Shape9DimType64E" title="tensorrt_llm::executor::Shape::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5Shape4BaseE" title="tensorrt_llm::executor::Shape::Base"><span class="n"><span class="pre">Base</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_type</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeEPK9DimType64N4Base9size_typeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor5Shape5ShapeENSt16initializer_listI9DimType64EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor5Shape5ShapeENSt16initializer_listI9DimType64EE"></span><span id="_CPPv2N12tensorrt_llm8executor5Shape5ShapeENSt16initializer_listI9DimType64EE"></span><span id="tensorrt_llm::executor::Shape::Shape__std::initializer_list:DimType64:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Shape_1a9211bd067fe9f04c94a49decf4b812de"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Shape</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5Shape9DimType64E" title="tensorrt_llm::executor::Shape::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeENSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6TensorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6TensorE"></span><span id="_CPPv2N12tensorrt_llm8executor6TensorE"></span><span id="tensorrt_llm::executor::Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Tensor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor13CudaStreamPtrE"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor13CudaStreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ab192218140e150441c3809208318577d"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor9copyToCpuEN6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor9copyToCpuEN6Tensor13CudaStreamPtrE"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor9copyToCpuEN6Tensor13CudaStreamPtrE"></span><span id="tensorrt_llm::executor::Tensor::copyToCpu__Tensor::CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a50aa2db7bc31318e493941918a304f6a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToCpu</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor9copyToCpuEN6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor12copyToPinnedEN6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor12copyToPinnedEN6Tensor13CudaStreamPtrE"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor12copyToPinnedEN6Tensor13CudaStreamPtrE"></span><span id="tensorrt_llm::executor::Tensor::copyToPinned__Tensor::CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a97a17d0f122678e4269d578f8bbcc057"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToPinned</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor12copyToPinnedEN6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor18copyToPooledPinnedEN6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor18copyToPooledPinnedEN6Tensor13CudaStreamPtrE"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor18copyToPooledPinnedEN6Tensor13CudaStreamPtrE"></span><span id="tensorrt_llm::executor::Tensor::copyToPooledPinned__Tensor::CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a772e702ed0364b8fe922b1488e06f42d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToPooledPinned</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor18copyToPooledPinnedEN6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor13copyToManagedEN6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor13copyToManagedEN6Tensor13CudaStreamPtrE"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor13copyToManagedEN6Tensor13CudaStreamPtrE"></span><span id="tensorrt_llm::executor::Tensor::copyToManaged__Tensor::CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a82d22baf219b3cf39e0ad7ea010e5457"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToManaged</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor13copyToManagedEN6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor9copyToGpuEN6Tensor13CudaStreamPtrE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor9copyToGpuEN6Tensor13CudaStreamPtrE"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor9copyToGpuEN6Tensor13CudaStreamPtrE"></span><span id="tensorrt_llm::executor::Tensor::copyToGpu__Tensor::CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1aab06656d158bf52a97fdb554273bedd7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToGpu</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor9copyToGpuEN6Tensor13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6TensorEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6TensorEv"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6TensorEv"></span><span id="tensorrt_llm::executor::Tensor::Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ae79dbd3ecc8694d366f5e2b67556a8cd"></span><span class="sig-name descname"><span class="n"><span class="pre">Tensor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6TensorD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6TensorD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor6TensorD0Ev"></span><span id="tensorrt_llm::executor::Tensor::~Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a28b3397651c85c4065e25d8d10d1c103"></span><span class="sig-name descname"><span class="n"><span class="pre">~Tensor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6TensorD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6TensorERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6TensorERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6TensorERK6Tensor"></span><span id="tensorrt_llm::executor::Tensor::Tensor__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a785597fe1ac8b56286d4ad836ec70035"></span><span class="sig-name descname"><span class="n"><span class="pre">Tensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERK6Tensor" title="tensorrt_llm::executor::Tensor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6TensorERR6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6TensorERR6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6TensorERR6Tensor"></span><span id="tensorrt_llm::executor::Tensor::Tensor__TensorRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ae40ad8a4bf115014f6ed52d674b07461"></span><span class="sig-name descname"><span class="n"><span class="pre">Tensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERR6Tensor" title="tensorrt_llm::executor::Tensor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERR6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6TensoraSERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6TensoraSERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6TensoraSERK6Tensor"></span><span id="tensorrt_llm::executor::Tensor::assign-operator__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a9a84b33d4e8f6106873b4cc03439534b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6TensoraSERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6TensoraSERR6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6TensoraSERR6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6TensoraSERR6Tensor"></span><span id="tensorrt_llm::executor::Tensor::assign-operator__TensorRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ad02487337036149be898932409f1f696"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6TensoraSERR6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor7getDataEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor7getDataEv"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor7getDataEv"></span><span id="tensorrt_llm::executor::Tensor::getData"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a420f7872623102fb87eab00b8df4a0d9"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getData</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor7getDataEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns a pointer to underlying array. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor7getDataEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor7getDataEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor7getDataEv"></span><span id="tensorrt_llm::executor::Tensor::getDataC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ab4e6ad75a64407f0fb8caa362098e3ab"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getData</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor7getDataEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns a pointer to underlying array. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor11getDataTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor11getDataTypeEv"></span><span id="tensorrt_llm::executor::Tensor::getDataTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a745614bf656424a3576690a25b5afbd4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the data type of the buffer. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor13getMemoryTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor13getMemoryTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor13getMemoryTypeEv"></span><span id="tensorrt_llm::executor::Tensor::getMemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ab60d4a2939254f259d6354714bb3396a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10MemoryTypeE" title="tensorrt_llm::executor::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMemoryType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor13getMemoryTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the memory type of the buffer. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor8getShapeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor8getShapeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor8getShapeEv"></span><span id="tensorrt_llm::executor::Tensor::getShapeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ab6f07c9a28710562e4366187b6d465cb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getShape</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor8getShapeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the tensor dimensions. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor7getSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor7getSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor7getSizeEv"></span><span id="tensorrt_llm::executor::Tensor::getSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a65ca052e56fa4b4b819456678a80cb82"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor7getSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the number of elements in the tensor. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor14getSizeInBytesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor14getSizeInBytesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor14getSizeInBytesEv"></span><span id="tensorrt_llm::executor::Tensor::getSizeInBytesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ae05d5b4735c1aeb2536903d51c2dc2fd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSizeInBytes</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor14getSizeInBytesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the size of the tensor in bytes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor7setZeroE13CudaStreamPtr">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor7setZeroE13CudaStreamPtr"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor7setZeroE13CudaStreamPtr"></span><span id="tensorrt_llm::executor::Tensor::setZero__CudaStreamPtr"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ad0cfa186ced1eb92039987e745aa5174"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setZero</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor7setZeroE13CudaStreamPtr" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Set the entire memory to zero. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>stream</strong> – Must be a valid CUDA stream if the memory type is GPU. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor7setFromERK6Tensor13CudaStreamPtr">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor7setFromERK6Tensor13CudaStreamPtr"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor7setFromERK6Tensor13CudaStreamPtr"></span><span id="tensorrt_llm::executor::Tensor::setFrom__TensorCR.CudaStreamPtr"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ab74daacf178945a17ecd8552ad6ea733"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFrom</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor7setFromERK6Tensor13CudaStreamPtr" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Copy the data and shape from another tensor. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>other</strong> – A tensor to copy from. </p></li>
|
||
<li><p><strong>stream</strong> – Must be a valid CUDA stream if the memory type is GPU. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6TensorcvbEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6TensorcvbEv"></span><span id="_CPPv2NK12tensorrt_llm8executor6TensorcvbEv"></span><span id="tensorrt_llm::executor::Tensor::castto-b-operatorC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a6c8f0a832643e66193ac9246423769ac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6TensorcvbEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6TensoreqERK6Tensor">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6TensoreqERK6Tensor"></span><span id="_CPPv2NK12tensorrt_llm8executor6TensoreqERK6Tensor"></span><span id="tensorrt_llm::executor::Tensor::eq-operator__TensorCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a80ff07f5b580df282b96a573c8e5bd5a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">rhs</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6TensoreqERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6TensorneERK6Tensor">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6TensorneERK6Tensor"></span><span id="_CPPv2NK12tensorrt_llm8executor6TensorneERK6Tensor"></span><span id="tensorrt_llm::executor::Tensor::neq-operator__TensorCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a35337e466b30dab925a4e895406214ac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">!=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">rhs</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6TensorneERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor3cpuE8DataType5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor3cpuE8DataType5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor3cpuE8DataType5Shape"></span><span id="tensorrt_llm::executor::Tensor::cpu__DataType.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ac116935be582b663662c6f39c7b774a5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cpu</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor3cpuE8DataType5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Allocate a cpu tensor with the given shape and data type.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor3cpuE6Tensor5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor3cpuE5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor3cpuE5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a7ddcb397b6da797ca7825e93841abf19"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cpu</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor3cpuE6Tensor5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6pinnedE8DataType5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6pinnedE8DataType5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6pinnedE8DataType5Shape"></span><span id="tensorrt_llm::executor::Tensor::pinned__DataType.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1aced97f5afb664a7dfc416a02b587462a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinned</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6pinnedE8DataType5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Allocate a cpu tensor in pinned memory with the given shape and data type.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor6pinnedE6Tensor5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor6pinnedE5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor6pinnedE5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1ada1a99697b667dd46221f73a9a9ee7b9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinned</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor6pinnedE6Tensor5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor12pooledPinnedE8DataType5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor12pooledPinnedE8DataType5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor12pooledPinnedE8DataType5Shape"></span><span id="tensorrt_llm::executor::Tensor::pooledPinned__DataType.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a7ba670c87ea79d7e66d562c22d7821ca"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pooledPinned</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor12pooledPinnedE8DataType5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Allocate a cpu tensor in pooled pinned memory with the given shape and data type.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor12pooledPinnedE6Tensor5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor12pooledPinnedE5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor12pooledPinnedE5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a66ac66e737e5481c8f1640f15ce613fc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pooledPinned</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor12pooledPinnedE6Tensor5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor7managedE8DataType5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor7managedE8DataType5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor7managedE8DataType5Shape"></span><span id="tensorrt_llm::executor::Tensor::managed__DataType.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a7731d566daed52054eccabfe3de81e1a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">managed</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor7managedE8DataType5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Allocate a tensor in managed memory (UVM) with the given shape and data type.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor7managedE6Tensor5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor7managedE5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor7managedE5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1aa95d4b6ccfcc20b47fb6a0145073c57d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">managed</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor7managedE6Tensor5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor3gpuE8DataType13CudaStreamPtr5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor3gpuE8DataType13CudaStreamPtr5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor3gpuE8DataType13CudaStreamPtr5Shape"></span><span id="tensorrt_llm::executor::Tensor::gpu__DataType.CudaStreamPtr.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a3fe1404ac1edb89baebaeb852817823d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpu</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor3gpuE8DataType13CudaStreamPtr5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Allocate a gpu tensor with the given shape and data type on a particular cuda stream.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>stream</strong> – Specifies the CUDA stream on which to allocate the tensor for GPU memory. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor3gpuE6Tensor13CudaStreamPtr5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor3gpuE13CudaStreamPtr5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor3gpuE13CudaStreamPtr5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a0bbc0df61a4c8c3187278b66f38f41fe"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpu</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor3gpuE6Tensor13CudaStreamPtr5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor2ofE8DataTypePv5Shape">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor2ofE8DataTypePv5Shape"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor2ofE8DataTypePv5Shape"></span><span id="tensorrt_llm::executor::Tensor::of__DataType.voidP.Shape"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a53fce3c66408ad1da79dc838ea9a1921"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">of</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor2ofE8DataTypePv5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Wrap a data pointer into a tensor without taking ownership.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
<li><p><strong>stream</strong> – Specifies the CUDA stream on which to allocate the tensor for GPU memory. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorP1T5Shape">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor2ofEP1T5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor2ofEP1T5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1abb953f082c26af69c559128e3e30b8ca"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">of</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorP1T5Shape" title="tensorrt_llm::executor::Tensor::of::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor5ShapeE" title="tensorrt_llm::executor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">shape</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorP1T5Shape" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Wrap a data pointer into a tensor without taking ownership.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
<li><p><strong>stream</strong> – Specifies the CUDA stream on which to allocate the tensor for GPU memory. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorR1T">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor2ofER1T"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor2ofER1T"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a367caf7b52fc7ccef5fe8e879376743b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">of</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorR1T" title="tensorrt_llm::executor::Tensor::of::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorR1T" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Wrap any container into a tensor without taking ownership.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
|
||
<li><p><strong>dataType</strong> – The data type of the tensor. </p></li>
|
||
<li><p><strong>stream</strong> – Specifies the CUDA stream on which to allocate the tensor for GPU memory. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor4ImplE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor4ImplE"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor4ImplE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a499dd2cff3f5f1ebd0a2a508462af729"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Impl</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor4ImplE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6TensorENSt10shared_ptrIN7runtime7ITensorEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6TensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6TensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="tensorrt_llm::executor::Tensor::Tensor__std::shared_ptr:runtime::ITensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1abaff3fb79d1193b805d9ed9182f9482f"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Tensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorENSt10shared_ptrIN7runtime7ITensorEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor6Tensor6copyToENSt10shared_ptrI4ImplEE13CudaStreamPtr">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor6Tensor6copyToENSt10shared_ptrI4ImplEE13CudaStreamPtr"></span><span id="_CPPv2NK12tensorrt_llm8executor6Tensor6copyToENSt10shared_ptrI4ImplEE13CudaStreamPtr"></span><span id="tensorrt_llm::executor::Tensor::copyTo__std::shared_ptr:Impl:.CudaStreamPtrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1adce5c2600b80e6aac03d72f62c442eaa"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyTo</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor4ImplE" title="tensorrt_llm::executor::Tensor::Impl"><span class="n"><span class="pre">Impl</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE" title="tensorrt_llm::executor::Tensor::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor6Tensor6copyToENSt10shared_ptrI4ImplEE13CudaStreamPtr" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor7mTensorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor7mTensorE"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor7mTensorE"></span><span id="tensorrt_llm::executor::Tensor::mTensor__std::shared_ptr:Impl:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1af5858fa2abb457b83fc8aa0cfa0392ce"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6Tensor4ImplE" title="tensorrt_llm::executor::Tensor::Impl"><span class="n"><span class="pre">Impl</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTensor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor7mTensorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor6Tensor14getRuntimeTypeE8DataTypev">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor6Tensor14getRuntimeTypeEv"></span><span id="_CPPv2I0EN12tensorrt_llm8executor6Tensor14getRuntimeTypeEv"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a323f2561238fa6191bf4b18b4916a464"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRuntimeType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor14getRuntimeTypeE8DataTypev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6detail9toITensorERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6detail9toITensorERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6detail9toITensorERK6Tensor"></span><span id="tensorrt_llm::executor::Tensor::detail::toITensor__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a6cf491a789adc04fcedfc1022a5167dd"></span><span class="k"><span class="pre">friend</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">toITensor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6detail9toITensorERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Tensor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Tensor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="_CPPv2N12tensorrt_llm8executor6Tensor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="tensorrt_llm::executor::Tensor::detail::ofITensor__std::shared_ptr:runtime::ITensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Tensor_1a8e986804106e3fe17218fbeebcb2659c"></span><span class="k"><span class="pre">friend</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ofITensor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Tensor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6detailE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6detailE"></span><span id="_CPPv2N12tensorrt_llm8executor6detailE"></span><span id="tensorrt_llm::executor::detail"></span><span class="target" id="namespacetensorrt__llm_1_1executor_1_1detail"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">detail</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6detailE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6detail9DimType64E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6detail9DimType64E"></span><span id="_CPPv2N12tensorrt_llm8executor6detail9DimType64E"></span><span class="target" id="tensor_8h_1aaa88c014789556bcf4f890ef4f91842b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DimType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">int64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6detail9DimType64E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6detail9toITensorERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6detail9toITensorERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor6detail9toITensorERK6Tensor"></span><span id="tensorrt_llm::executor::detail::toITensor__TensorCR"></span><span class="target" id="tensor_8h_1a688a8726508dd792068204eb5cf8273b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">toITensor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6detail9toITensorERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="_CPPv2N12tensorrt_llm8executor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"></span><span id="tensorrt_llm::executor::detail::ofITensor__std::shared_ptr:runtime::ITensor:"></span><span class="target" id="tensor_8h_1acb29416de14060f4546f8653af9220e8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ofITensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimeE">
|
||
<span id="_CPPv3N12tensorrt_llm7runtimeE"></span><span id="_CPPv2N12tensorrt_llm7runtimeE"></span><span id="tensorrt_llm::runtime"></span><span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="transferagent-h">
|
||
<h2>transferAgent.h<a class="headerlink" href="#transferagent-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cacheE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cacheE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cacheE"></span><span id="tensorrt_llm::executor::kv_cache"></span><span class="target" id="namespacetensorrt__llm_1_1executor_1_1kv__cache"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kv_cache</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache13TransferDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache13TransferDescsE"></span><span class="target" id="transferAgent_8h_1ad1f49c49bb08248e8cd955df8292fbae"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TransferDescs</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE" title="tensorrt_llm::executor::kv_cache::MemoryDescs"><span class="n"><span class="pre">MemoryDescs</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache13RegisterDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache13RegisterDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache13RegisterDescsE"></span><span class="target" id="transferAgent_8h_1a2bb86b812372815ec90e52e4d9a17099"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RegisterDescs</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE" title="tensorrt_llm::executor::kv_cache::MemoryDescs"><span class="n"><span class="pre">MemoryDescs</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13RegisterDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11SyncMessageE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11SyncMessageE"></span><span class="target" id="transferAgent_8h_1a2286881f67c6a7048094b5b611741cfc"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SyncMessage</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE"></span><span class="target" id="transferAgent_8h_1ac763f2223d964bea6fc3424ea1e66896"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ConnectionInfoType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-enums">Enums</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryTypeE"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039a"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kDRAME">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryType5kDRAME"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryType5kDRAME"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039aa42114399bc430c192559868559876494"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDRAM</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kDRAME" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kVRAME">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryType5kVRAME"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryType5kVRAME"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039aa1553fa1962a86fec3af0c6d1f2cb34f0"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kVRAM</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kVRAME" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kBLKE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryType4kBLKE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryType4kBLKE"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039aa9a08b41ce2bbaa0878f2b23970ab01b0"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBLK</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kBLKE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kOBJE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryType4kOBJE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryType4kOBJE"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039aa04ae6375ff7dd60354d217d706198112"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kOBJ</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kOBJE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kFILEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryType5kFILEE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryType5kFILEE"></span><span class="target" id="transferAgent_8h_1a3c82e9ebcab35b8ab4d39e16f4f9039aa4f5e30bd18513f0849246100edf4b267"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kFILE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kFILEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10TransferOpE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10TransferOpE"></span><span class="target" id="transferAgent_8h_1a3e6174d68fd0641f72787ca2b45a0fee"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TransferOp</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp5kREADE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10TransferOp5kREADE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10TransferOp5kREADE"></span><span class="target" id="transferAgent_8h_1a3e6174d68fd0641f72787ca2b45a0feea8fbb854b62e34a1f77d600f286f5d449"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kREAD</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp5kREADE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp6kWRITEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10TransferOp6kWRITEE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10TransferOp6kWRITEE"></span><span class="target" id="transferAgent_8h_1a3e6174d68fd0641f72787ca2b45a0feea9b3031051108ec0a493a1c56c664a6f3"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kWRITE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp6kWRITEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentENSt10unique_ptrI17BaseTransferAgentEERKNSt6stringEDpRR4Args">
|
||
<span id="_CPPv3IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentERKNSt6stringEDpRR4Args"></span><span id="_CPPv2IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentERKNSt6stringEDpRR4Args"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="p"><span class="pre">...</span></span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">Args</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="transferAgent_8h_1a962b2ba4955f3a9e8f0da6eaca718077"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE" title="tensorrt_llm::executor::kv_cache::BaseTransferAgent"><span class="n"><span class="pre">BaseTransferAgent</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">makeTransferAgent</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">backend</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentENSt10unique_ptrI17BaseTransferAgentEERKNSt6stringEDpRR4Args" title="tensorrt_llm::executor::kv_cache::makeTransferAgent::Args"><span class="n"><span class="pre">Args</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">...</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">args</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentENSt10unique_ptrI17BaseTransferAgentEERKNSt6stringEDpRR4Args" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentENSt10shared_ptrI17BaseLoopbackAgentEERKNSt6stringEDpRR4Args">
|
||
<span id="_CPPv3IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentERKNSt6stringEDpRR4Args"></span><span id="_CPPv2IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentERKNSt6stringEDpRR4Args"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="p"><span class="pre">...</span></span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">Args</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="transferAgent_8h_1a0b866b77a2f3f093de5dad682fcf82c6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE" title="tensorrt_llm::executor::kv_cache::BaseLoopbackAgent"><span class="n"><span class="pre">BaseLoopbackAgent</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">makeLoopbackAgent</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">backend</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentENSt10shared_ptrI17BaseLoopbackAgentEERKNSt6stringEDpRR4Args" title="tensorrt_llm::executor::kv_cache::makeLoopbackAgent::Args"><span class="n"><span class="pre">Args</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">...</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">args</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentENSt10shared_ptrI17BaseLoopbackAgentEERKNSt6stringEDpRR4Args" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDescE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9AgentDescE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9AgentDescE"></span><span id="tensorrt_llm::executor::kv_cache::AgentDesc"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentDesc"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AgentDesc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDescE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc9AgentDescENSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9AgentDesc9AgentDescENSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9AgentDesc9AgentDescENSt6stringE"></span><span id="tensorrt_llm::executor::kv_cache::AgentDesc::AgentDesc__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentDesc_1a48dcdf4866378fb0c32b23ba8af5579a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AgentDesc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">backendAgentDesc</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc9AgentDescENSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9AgentDesc19getBackendAgentDescEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9AgentDesc19getBackendAgentDescEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9AgentDesc19getBackendAgentDescEv"></span><span id="tensorrt_llm::executor::kv_cache::AgentDesc::getBackendAgentDescC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentDesc_1a14f0346b57a5ad1b9f609a1e1c96c0ae"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getBackendAgentDesc</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9AgentDesc19getBackendAgentDescEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc17mBackendAgentDescE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9AgentDesc17mBackendAgentDescE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9AgentDesc17mBackendAgentDescE"></span><span id="tensorrt_llm::executor::kv_cache::AgentDesc::mBackendAgentDesc__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentDesc_1ae3fc107c8657064a17abac3b3f0f585f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBackendAgentDesc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc17mBackendAgentDescE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15BaseAgentConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15BaseAgentConfigE"></span><span id="tensorrt_llm::executor::kv_cache::BaseAgentConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseAgentConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BaseAgentConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig5mNameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15BaseAgentConfig5mNameE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15BaseAgentConfig5mNameE"></span><span id="tensorrt_llm::executor::kv_cache::BaseAgentConfig::mName__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseAgentConfig_1a2936d275df3a561da7588cd2c1cf28ec"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mName</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig5mNameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig13useProgThreadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15BaseAgentConfig13useProgThreadE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15BaseAgentConfig13useProgThreadE"></span><span id="tensorrt_llm::executor::kv_cache::BaseAgentConfig::useProgThread__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseAgentConfig_1a314e831a12e6b318d60425b3dc699813"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useProgThread</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig13useProgThreadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig11multiThreadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15BaseAgentConfig11multiThreadE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15BaseAgentConfig11multiThreadE"></span><span id="tensorrt_llm::executor::kv_cache::BaseAgentConfig::multiThread__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseAgentConfig_1a4ccd5e49c707f8d7cc3e8cd95e2136ad"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">multiThread</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig11multiThreadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE"></span><span id="tensorrt_llm::executor::kv_cache::BaseLoopbackAgent"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseLoopbackAgent"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BaseLoopbackAgent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::BaseLoopbackAgent::~BaseLoopbackAgent"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseLoopbackAgent_1aa3c3c770746a385a1cbc1292f2dda94c"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~BaseLoopbackAgent</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgent22executeLoopbackRequestERK11MemoryDescsRK9FileDescsb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgent22executeLoopbackRequestERK11MemoryDescsRK9FileDescsb"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgent22executeLoopbackRequestERK11MemoryDescsRK9FileDescsb"></span><span id="tensorrt_llm::executor::kv_cache::BaseLoopbackAgent::executeLoopbackRequest__MemoryDescsCR.FileDescsCR.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseLoopbackAgent_1a9a47add4a8e31dfd45a9be6e097b17cf"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executeLoopbackRequest</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE" title="tensorrt_llm::executor::kv_cache::MemoryDescs"><span class="n"><span class="pre">MemoryDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">memoryDescs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescsE" title="tensorrt_llm::executor::kv_cache::FileDescs"><span class="n"><span class="pre">FileDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">fileDescs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isOffload</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgent22executeLoopbackRequestERK11MemoryDescsRK9FileDescsb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BaseTransferAgent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgentD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgentD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::~BaseTransferAgent"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1ac38476f4d667e6959a62992548730e72"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~BaseTransferAgent</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent14registerMemoryERK13RegisterDescs">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent14registerMemoryERK13RegisterDescs"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent14registerMemoryERK13RegisterDescs"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::registerMemory__RegisterDescsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1af2ab4f613dbe8856dc215e64f327136a"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">registerMemory</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13RegisterDescsE" title="tensorrt_llm::executor::kv_cache::RegisterDescs"><span class="n"><span class="pre">RegisterDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">descs</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent14registerMemoryERK13RegisterDescs" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Register a memory region. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>descs</strong> – Describe the memory regions to be registered. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16deregisterMemoryERK13RegisterDescs">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16deregisterMemoryERK13RegisterDescs"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16deregisterMemoryERK13RegisterDescs"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::deregisterMemory__RegisterDescsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1aa55facd04a0995f9f1837db545b4cb94"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deregisterMemory</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13RegisterDescsE" title="tensorrt_llm::executor::kv_cache::RegisterDescs"><span class="n"><span class="pre">RegisterDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">descs</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16deregisterMemoryERK13RegisterDescs" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Unregister a memory region. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>descs</strong> – Describe the memory regions to be unregistered. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK9AgentDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK9AgentDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK9AgentDesc"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::loadRemoteAgent__ssCR.AgentDescCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1a1447916398ed57751cb0773875e35b55"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loadRemoteAgent</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDescE" title="tensorrt_llm::executor::kv_cache::AgentDesc"><span class="n"><span class="pre">AgentDesc</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">agentDesc</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK9AgentDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Initialize and establish a connection with a remote agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>name</strong> – Specify the name of the remote agent. </p></li>
|
||
<li><p><strong>agentDesc</strong> – Provide the necessary communication details for connecting to the remote agent. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK18ConnectionInfoType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK18ConnectionInfoType"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK18ConnectionInfoType"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::loadRemoteAgent__ssCR.ConnectionInfoTypeCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1aa07eb6026171846681b7887c6f912db3"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loadRemoteAgent</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE" title="tensorrt_llm::executor::kv_cache::ConnectionInfoType"><span class="n"><span class="pre">ConnectionInfoType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">connectionInfo</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK18ConnectionInfoType" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Initialize and establish a connection with a remote agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>name</strong> – Specify the name of the remote agent. </p></li>
|
||
<li><p><strong>connectionInfo</strong> – Provide the necessary communication details for connecting to the remote agent. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent21invalidateRemoteAgentERKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent21invalidateRemoteAgentERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent21invalidateRemoteAgentERKNSt6stringE"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::invalidateRemoteAgent__ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1ac347f34b38bb87755efe08b7d64bb01c"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">invalidateRemoteAgent</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent21invalidateRemoteAgentERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Invalidate a connection with a remote agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>name</strong> – Specify the name of the remote agent. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17getLocalAgentDescEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17getLocalAgentDescEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17getLocalAgentDescEv"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::getLocalAgentDesc"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1abd9748104966f66cde9a1be618487abb"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDescE" title="tensorrt_llm::executor::kv_cache::AgentDesc"><span class="n"><span class="pre">AgentDesc</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLocalAgentDesc</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17getLocalAgentDescEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Fetch the descriptor of the local agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>The descriptor of the local agent. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22getLocalConnectionInfoEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22getLocalConnectionInfoEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22getLocalConnectionInfoEv"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::getLocalConnectionInfo"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1a213583a0dfe31271ce37979307413b4b"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE" title="tensorrt_llm::executor::kv_cache::ConnectionInfoType"><span class="n"><span class="pre">ConnectionInfoType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLocalConnectionInfo</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22getLocalConnectionInfoEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Fetch the descriptor of the local agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>The descriptor of the local agent. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22submitTransferRequestsERK15TransferRequest">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22submitTransferRequestsERK15TransferRequest"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22submitTransferRequestsERK15TransferRequest"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::submitTransferRequests__TransferRequestCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1aebf9717ab007f261b7006197de0bee73"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusE" title="tensorrt_llm::executor::kv_cache::TransferStatus"><span class="n"><span class="pre">TransferStatus</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">submitTransferRequests</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequestE" title="tensorrt_llm::executor::kv_cache::TransferRequest"><span class="n"><span class="pre">TransferRequest</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">request</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22submitTransferRequestsERK15TransferRequest" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Initiate the transfer by submitting the request. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>request</strong> – Specify the transmission request. </p>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>The status of the requests. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17notifySyncMessageERKNSt6stringERK11SyncMessage">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17notifySyncMessageERKNSt6stringERK11SyncMessage"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17notifySyncMessageERKNSt6stringERK11SyncMessage"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::notifySyncMessage__ssCR.SyncMessageCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1a5fa37e2a12de2bb6de39c5ac57b1a020"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">notifySyncMessage</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="tensorrt_llm::executor::kv_cache::SyncMessage"><span class="n"><span class="pre">SyncMessage</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">syncMessage</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17notifySyncMessageERKNSt6stringERK11SyncMessage" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Generate a notification, not bound to a transfer, e.g., for control. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>name</strong> – Specify the name of the remote agent to which the information should be sent. </p></li>
|
||
<li><p><strong>syncMessage</strong> – The data or message intended for synchronization. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent23getNotifiedSyncMessagesEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent23getNotifiedSyncMessagesEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent23getNotifiedSyncMessagesEv"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::getNotifiedSyncMessages"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1a8b84bb623ba08c93c850f7909e866441"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="tensorrt_llm::executor::kv_cache::SyncMessage"><span class="n"><span class="pre">SyncMessage</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNotifiedSyncMessages</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent23getNotifiedSyncMessagesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Retrieve notification messages sent by other agents. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>A mapping from remote agent names to their respective notification messages. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16checkRemoteDescsERKNSt6stringERK11MemoryDescs">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16checkRemoteDescsERKNSt6stringERK11MemoryDescs"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16checkRemoteDescsERKNSt6stringERK11MemoryDescs"></span><span id="tensorrt_llm::executor::kv_cache::BaseTransferAgent::checkRemoteDescs__ssCR.MemoryDescsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1BaseTransferAgent_1a2b391691d49d70cb97915f3d336d6ef3"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">checkRemoteDescs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE" title="tensorrt_llm::executor::kv_cache::MemoryDescs"><span class="n"><span class="pre">MemoryDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">memoryDescs</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16checkRemoteDescsERKNSt6stringERK11MemoryDescs" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Check if metadata is available for a remote agent. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>Whether the metadata is available for a remote agent. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoaderE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoaderE"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DynLibLoader</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9getHandleERKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader9getHandleERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader9getHandleERKNSt6stringE"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::getHandle__ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1ac53d5bc596a947fa23a4b223bd6e96ad"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getHandle</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">name</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9getHandleERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerE9FunctionTRKNSt6stringERKNSt6stringE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerERKNSt6stringERKNSt6stringE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerERKNSt6stringERKNSt6stringE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">FunctionT</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1aa120a1793a0add730f8f8a3b4a3fdb02"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerE9FunctionTRKNSt6stringERKNSt6stringE" title="tensorrt_llm::executor::kv_cache::DynLibLoader::getFunctionPointer::FunctionT"><span class="n"><span class="pre">FunctionT</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFunctionPointer</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">libName</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">funcName</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerE9FunctionTRKNSt6stringERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoaderD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoaderD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::~DynLibLoader"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1afc8ec9ba9f94e6f4d9f92dad576ef78c"></span><span class="sig-name descname"><span class="n"><span class="pre">~DynLibLoader</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderEv"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::DynLibLoader"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1a7fef19629812725c387457b230b2a18b"></span><span class="sig-name descname"><span class="n"><span class="pre">DynLibLoader</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::DynLibLoader__DynLibLoaderCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1a40ba67066154f109542a91dc8dc53224"></span><span class="sig-name descname"><span class="n"><span class="pre">DynLibLoader</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader" title="tensorrt_llm::executor::kv_cache::DynLibLoader::DynLibLoader"><span class="n"><span class="pre">DynLibLoader</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderaSERK12DynLibLoader">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoaderaSERK12DynLibLoader"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoaderaSERK12DynLibLoader"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::assign-operator__DynLibLoaderCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1af753fc3984edb13b76ed8c2a3d4c0e95"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE" title="tensorrt_llm::executor::kv_cache::DynLibLoader"><span class="n"><span class="pre">DynLibLoader</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE" title="tensorrt_llm::executor::kv_cache::DynLibLoader"><span class="n"><span class="pre">DynLibLoader</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderaSERK12DynLibLoader" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader11getInstanceEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader11getInstanceEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader11getInstanceEv"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::getInstance"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1a536b9e15fe4aac0e3e3965376f9e7655"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE" title="tensorrt_llm::executor::kv_cache::DynLibLoader"><span class="n"><span class="pre">DynLibLoader</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getInstance</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader11getInstanceEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mDllMutexE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader9mDllMutexE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader9mDllMutexE"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::mDllMutex__std::mutex"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1a8fde9ddc597323cbf44e3374b352cdb9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">mutex</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDllMutex</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mDllMutexE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mHandlersE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader9mHandlersE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader9mHandlersE"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::mHandlers__std::unordered_map:ss.voidP:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1aa62953ffd11b8b0094a999170bcb964b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mHandlers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mHandlersE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader5dlSymEPvPKc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache12DynLibLoader5dlSymEPvPKc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache12DynLibLoader5dlSymEPvPKc"></span><span id="tensorrt_llm::executor::kv_cache::DynLibLoader::dlSym__voidP.cCP"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1DynLibLoader_1a968ec20ae0e3b5aa0c2d138b66f299ff"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">dlSym</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">handle</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">symbol</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader5dlSymEPvPKc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDescE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDescE"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FileDesc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERKNSt6stringEi6mode_t6size_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERKNSt6stringEi6mode_t6size_t"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERKNSt6stringEi6mode_t6size_t"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::FileDesc__ssCR.i.mode_t.s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1abe358dda0224d6a9f6d8bd495d2ed84c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FileDesc</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">filename</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">flags</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">mode_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">len</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERKNSt6stringEi6mode_t6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::FileDesc__FileDescRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1a1d895646e327c6cab1a04a3a262d7750"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FileDesc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc" title="tensorrt_llm::executor::kv_cache::FileDesc::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERR8FileDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDescaSERR8FileDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDescaSERR8FileDesc"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::assign-operator__FileDescRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1aaed126f497bdcb2bd843055590f84628"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERR8FileDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDescD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDescD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::~FileDesc"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1ae320404b2a99cefd344dc0f804a65547"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~FileDesc</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc5getFdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache8FileDesc5getFdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache8FileDesc5getFdEv"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::getFdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1aebe36a9e350f4bc769681464fbcabaab"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFd</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc5getFdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc6getLenEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache8FileDesc6getLenEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache8FileDesc6getLenEv"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::getLenC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1a53e498988352985cda8fa864a148cc66"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc6getLenEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::FileDesc__FileDescCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1a9a5cb48980328a8a6d8e93d950023394"></span><span class="sig-name descname"><span class="n"><span class="pre">FileDesc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc" title="tensorrt_llm::executor::kv_cache::FileDesc::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERK8FileDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDescaSERK8FileDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDescaSERK8FileDesc"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::assign-operator__FileDescCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1a86167ce814f4204e7797263bbc146cc3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERK8FileDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc2fdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDesc2fdE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDesc2fdE"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::fd__i"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1a83202d562474d79d8000c7e0b523e4ae"></span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fd</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc2fdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc4mLenE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8FileDesc4mLenE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8FileDesc4mLenE"></span><span id="tensorrt_llm::executor::kv_cache::FileDesc::mLen__s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDesc_1af3eed784d859784440f5da9c8f152724"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc4mLenE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9FileDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9FileDescsE"></span><span id="tensorrt_llm::executor::kv_cache::FileDescs"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDescs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FileDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs9FileDescsERRNSt6vectorI8FileDescEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9FileDescs9FileDescsERRNSt6vectorI8FileDescEE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9FileDescs9FileDescsERRNSt6vectorI8FileDescEE"></span><span id="tensorrt_llm::executor::kv_cache::FileDescs::FileDescs__std::vector:FileDesc:RR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDescs_1af3696f083c511093d201eb5aada6fc5c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FileDescs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">descs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs9FileDescsERRNSt6vectorI8FileDescEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9FileDescs8getDescsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9FileDescs8getDescsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9FileDescs8getDescsEv"></span><span id="tensorrt_llm::executor::kv_cache::FileDescs::getDescsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDescs_1a8c33cd53a5c346d96e9c8e50ba96c189"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDescs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9FileDescs8getDescsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs6mDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9FileDescs6mDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9FileDescs6mDescsE"></span><span id="tensorrt_llm::executor::kv_cache::FileDescs::mDescs__std::vector:FileDesc:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1FileDescs_1a6d7e19ecff9827aa344fa4e0ed0ebf2e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE" title="tensorrt_llm::executor::kv_cache::FileDesc"><span class="n"><span class="pre">FileDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs6mDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDescE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDescE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDesc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescERKNSt6vectorIcEE8uint32_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescERKNSt6vectorIcEE8uint32_t"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescERKNSt6vectorIcEE8uint32_t"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::MemoryDesc__std::vector:c:CR.uint32_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a2a0d8735dd403faea98e2774904ae876"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDesc</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">vec</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">uint32_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">deviceId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescERKNSt6vectorIcEE8uint32_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescEPv6size_t8uint32_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescEPv6size_t8uint32_t"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescEPv6size_t8uint32_t"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::MemoryDesc__voidP.s.uint32_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1ad9be073c41d131586b2f83096ea5ed42"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDesc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">addr</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">len</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">uint32_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">deviceId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescEPv6size_t8uint32_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescE9uintptr_t6size_t8uint32_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescE9uintptr_t6size_t8uint32_t"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescE9uintptr_t6size_t8uint32_t"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::MemoryDesc__uintptr_t.s.uint32_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a02deebfb2875dc0ad55524ea456c5beb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDesc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">uintptr_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">addr</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">len</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">uint32_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">deviceId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescE9uintptr_t6size_t8uint32_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc7getAddrEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10MemoryDesc7getAddrEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10MemoryDesc7getAddrEv"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::getAddrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a4a74dbbcf3978170afa7d01070084041"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">uintptr_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAddr</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc7getAddrEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc6getLenEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10MemoryDesc6getLenEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10MemoryDesc6getLenEv"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::getLenC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a41dbdc2f221c6f79b3b5570ecfff5b60"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc6getLenEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc11getDeviceIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10MemoryDesc11getDeviceIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10MemoryDesc11getDeviceIdEv"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::getDeviceIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a0133ed4bdf8ffd4323d335b7fe530e8a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">uint32_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDeviceId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc11getDeviceIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9serializeERK10MemoryDescRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc9serializeERK10MemoryDescRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc9serializeERK10MemoryDescRNSt7ostreamE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::serialize__MemoryDescCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a3a98dd704a4bf7023c32032a69182558"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">memoryDesc</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9serializeERK10MemoryDescRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc11deserializeERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc11deserializeERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc11deserializeERNSt7istreamE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::deserialize__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a097411ed09a8a12dcaee26bbed268764"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc11deserializeERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc14serializedSizeERK10MemoryDesc">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc14serializedSizeERK10MemoryDesc"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc14serializedSizeERK10MemoryDesc"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::serializedSize__MemoryDescCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a195e62a86d381e190e1525306a240890"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">memoryDesc</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc14serializedSizeERK10MemoryDesc" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc5mAddrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc5mAddrE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc5mAddrE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::mAddr__uintptr_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a4e60eb382918f123f11e6db8fdb3c943"></span><span class="n"><span class="pre">uintptr_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAddr</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc5mAddrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc4mLenE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc4mLenE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc4mLenE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::mLen__s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a4776ae22b3922505e55eaf4f278d5143"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc4mLenE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9mDeviceIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10MemoryDesc9mDeviceIdE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10MemoryDesc9mDeviceIdE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDesc::mDeviceId__uint32_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDesc_1a3365a3c18600915e57e9e034cef567ee"></span><span class="n"><span class="pre">uint32_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDeviceId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9mDeviceIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11MemoryDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11MemoryDescsE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs11MemoryDescsE10MemoryTypeNSt6vectorI10MemoryDescEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11MemoryDescs11MemoryDescsE10MemoryTypeNSt6vectorI10MemoryDescEE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11MemoryDescs11MemoryDescsE10MemoryTypeNSt6vectorI10MemoryDescEE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs::MemoryDescs__MemoryType.std::vector:MemoryDesc:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs_1a8295bed464d811c027ce4691a0e15cd6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryDescs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE" title="tensorrt_llm::executor::kv_cache::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">descs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs11MemoryDescsE10MemoryTypeNSt6vectorI10MemoryDescEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs7getTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache11MemoryDescs7getTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache11MemoryDescs7getTypeEv"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs::getTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs_1a111f124275f834d2387b2df5432b71a9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE" title="tensorrt_llm::executor::kv_cache::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs7getTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs8getDescsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache11MemoryDescs8getDescsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache11MemoryDescs8getDescsEv"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs::getDescsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs_1a4e42d94b90a4a5b95e896c533721ae1b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDescs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs8getDescsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs5mTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11MemoryDescs5mTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11MemoryDescs5mTypeE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs::mType__MemoryType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs_1afe754835f089dd28d67bec3db8c79518"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE" title="tensorrt_llm::executor::kv_cache::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs5mTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs6mDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11MemoryDescs6mDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11MemoryDescs6mDescsE"></span><span id="tensorrt_llm::executor::kv_cache::MemoryDescs::mDescs__std::vector:MemoryDesc:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1MemoryDescs_1ae7d74ba13fb6f4f05c72609162553738"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE" title="tensorrt_llm::executor::kv_cache::MemoryDesc"><span class="n"><span class="pre">MemoryDesc</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs6mDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequestE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TransferRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest15TransferRequestE10TransferOp13TransferDescs13TransferDescsRKNSt6stringENSt8optionalI11SyncMessageEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest15TransferRequestE10TransferOp13TransferDescs13TransferDescsRKNSt6stringENSt8optionalI11SyncMessageEE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest15TransferRequestE10TransferOp13TransferDescs13TransferDescsRKNSt6stringENSt8optionalI11SyncMessageEE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::TransferRequest__TransferOp.TransferDescs.TransferDescs.ssCR.std::optional:SyncMessage:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a3f6d832fe6fba6180aaac43a08b8c262"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TransferRequest</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE" title="tensorrt_llm::executor::kv_cache::TransferOp"><span class="n"><span class="pre">TransferOp</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">op</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">srcDescs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dstDescs</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">remoteName</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="tensorrt_llm::executor::kv_cache::SyncMessage"><span class="n"><span class="pre">SyncMessage</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">syncMessage</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest15TransferRequestE10TransferOp13TransferDescs13TransferDescsRKNSt6stringENSt8optionalI11SyncMessageEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The constructor of <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest"><span class="std std-ref"><span class="pre">TransferRequest</span></span></a></code>. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>op</strong> – Source data arrangement. </p></li>
|
||
<li><p><strong>srcDescs</strong> – Description of the source memory region. </p></li>
|
||
<li><p><strong>dstDescs</strong> – Description of the destination memory region. </p></li>
|
||
<li><p><strong>remoteName</strong> – Name of the remote counterpart. </p></li>
|
||
<li><p><strong>syncMessage</strong> – Synchronization information for the end of the transfer. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest5getOpEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache15TransferRequest5getOpEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache15TransferRequest5getOpEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::getOpC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1ac533b6c1c1b8c5397ce8e25833b26158"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE" title="tensorrt_llm::executor::kv_cache::TransferOp"><span class="n"><span class="pre">TransferOp</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOp</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest5getOpEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getSrcDescsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache15TransferRequest11getSrcDescsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache15TransferRequest11getSrcDescsEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::getSrcDescsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a6a6eb8487a43ecb153502a7a09dad96e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSrcDescs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getSrcDescsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getDstDescsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache15TransferRequest11getDstDescsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache15TransferRequest11getDstDescsEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::getDstDescsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1ac86417f2f0dcd9dbdfc71c9db133b879"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDstDescs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getDstDescsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest13getRemoteNameEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache15TransferRequest13getRemoteNameEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache15TransferRequest13getRemoteNameEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::getRemoteNameC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1ace277e9971c3d7a09074d818324bfb71"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getRemoteName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest13getRemoteNameEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest14getSyncMessageEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache15TransferRequest14getSyncMessageEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache15TransferRequest14getSyncMessageEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::getSyncMessageC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a8d8c48b778b7abb203f545502d280399"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="tensorrt_llm::executor::kv_cache::SyncMessage"><span class="n"><span class="pre">SyncMessage</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSyncMessage</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest14getSyncMessageEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest3mOpE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest3mOpE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest3mOpE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::mOp__TransferOp"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a206d45cdbe53b9a4f280c901b51557f3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE" title="tensorrt_llm::executor::kv_cache::TransferOp"><span class="n"><span class="pre">TransferOp</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOp</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest3mOpE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mSrcDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest9mSrcDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest9mSrcDescsE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::mSrcDescs__TransferDescs"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1abce69416e78057dc63235fefd45e7cdb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSrcDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mSrcDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mDstDescsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest9mDstDescsE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest9mDstDescsE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::mDstDescs__TransferDescs"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a1843d8b65374bbe93e8c6d05ead25059"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE" title="tensorrt_llm::executor::kv_cache::TransferDescs"><span class="n"><span class="pre">TransferDescs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDstDescs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mDstDescsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest11mRemoteNameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest11mRemoteNameE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest11mRemoteNameE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::mRemoteName__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1ac7a5fcb8ee1ec8505f8057fdf1b69339"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRemoteName</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest11mRemoteNameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest12mSyncMessageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache15TransferRequest12mSyncMessageE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache15TransferRequest12mSyncMessageE"></span><span id="tensorrt_llm::executor::kv_cache::TransferRequest::mSyncMessage__std::optional:SyncMessage:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferRequest_1a4e3eb7e8611e553a56c30ea472821854"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE" title="tensorrt_llm::executor::kv_cache::SyncMessage"><span class="n"><span class="pre">SyncMessage</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSyncMessage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest12mSyncMessageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache14TransferStatusE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache14TransferStatusE"></span><span id="tensorrt_llm::executor::kv_cache::TransferStatus"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferStatus"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TransferStatus</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache14TransferStatusD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache14TransferStatusD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::TransferStatus::~TransferStatus"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferStatus_1a5875c08c018ed556bbb048bd71d4667a"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~TransferStatus</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus11isCompletedEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache14TransferStatus11isCompletedEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache14TransferStatus11isCompletedEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferStatus::isCompletedC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferStatus_1a0855f8e280bf6d0357c22a08d7cb79a5"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isCompleted</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus11isCompletedEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus4waitEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache14TransferStatus4waitEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache14TransferStatus4waitEv"></span><span id="tensorrt_llm::executor::kv_cache::TransferStatus::waitC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1TransferStatus_1a3295b58ae616e14c205b802e719c8b15"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wait</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus4waitEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="serialization-h">
|
||
<h2>serialization.h<a class="headerlink" href="#serialization-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13SerializationE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13SerializationE"></span><span id="_CPPv2N12tensorrt_llm8executor13SerializationE"></span><span id="tensorrt_llm::executor::Serialization"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Serialization</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13SerializationE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__tensorrt_llm::batch_manager::kv_cache_manager::BlockKeyCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a882380be63503183dd87a617932439f3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="tensorrt_llm::batch_manager::kv_cache_manager"><span class="n"><span class="pre">kv_cache_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">BlockKey</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">key</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__tensorrt_llm::batch_manager::kv_cache_manager::BlockKeyCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac2abb214965739a91a56d68984f211e8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="tensorrt_llm::batch_manager::kv_cache_manager"><span class="n"><span class="pre">kv_cache_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">BlockKey</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">key</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization19deserializeBlockKeyERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization19deserializeBlockKeyERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization19deserializeBlockKeyERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeBlockKey__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad62f4cb07b3b6b62233315f5367f5411"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="tensorrt_llm::batch_manager::kv_cache_manager"><span class="n"><span class="pre">kv_cache_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">BlockKey</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeBlockKey</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization19deserializeBlockKeyERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization20deserializeTimePointERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization20deserializeTimePointERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization20deserializeTimePointERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeTimePoint__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2d477b7cbaa81ccdce8b0228da633407"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeTimePoint</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeTimePointERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN18RequestPerfMetrics9TimePointERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN18RequestPerfMetrics9TimePointERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN18RequestPerfMetrics9TimePointERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestPerfMetrics::TimePointCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ada4597a81d9f13a07a92924a0d887444"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tp</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN18RequestPerfMetrics9TimePointERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN18RequestPerfMetrics9TimePointE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN18RequestPerfMetrics9TimePointE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN18RequestPerfMetrics9TimePointE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestPerfMetrics::TimePointCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a47205a5c23884ff19818607562d9565c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN18RequestPerfMetrics9TimePointE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeRequestPerfMetricsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeRequestPerfMetricsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeRequestPerfMetricsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestPerfMetrics__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a75fbc151569f640c8867ccea7f154284"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestPerfMetrics</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeRequestPerfMetricsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18RequestPerfMetricsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18RequestPerfMetricsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18RequestPerfMetricsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestPerfMetricsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a7780e617369c0cafdc7218ab69455499"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">metrics</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18RequestPerfMetricsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18RequestPerfMetrics">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18RequestPerfMetrics"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18RequestPerfMetrics"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestPerfMetricsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9727f8f81dbc73c2f57c06ac18392b91"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">metrics</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18RequestPerfMetrics" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeSamplingConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeSamplingConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeSamplingConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSamplingConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a018e2c20ed62e05f0428c770990cf3a7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSamplingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeSamplingConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14SamplingConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14SamplingConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14SamplingConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__SamplingConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab250085b9f35d5c2ca33e63241f4ffa5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14SamplingConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14SamplingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK14SamplingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK14SamplingConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__SamplingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2b0d1cd68b238760ff02f8a4740bead3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14SamplingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeOutputConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeOutputConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeOutputConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeOutputConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1afa9afb4d8f345960ce3419aa50a7aecb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeOutputConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeOutputConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12OutputConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12OutputConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12OutputConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__OutputConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a50563b0a86ded8ca3f7273d126ac7042"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12OutputConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12OutputConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12OutputConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12OutputConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__OutputConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aa51e9d72a4d69d15f2371d2eb8cbeba3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12OutputConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization32deserializeAdditionalModelOutputERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization32deserializeAdditionalModelOutputERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization32deserializeAdditionalModelOutputERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeAdditionalModelOutput__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a824074582eb598455769102520cef428"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeAdditionalModelOutput</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeAdditionalModelOutputERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21AdditionalModelOutputRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK21AdditionalModelOutputRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK21AdditionalModelOutputRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__AdditionalModelOutputCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1317bac076a4fc9cbf5098b2fb9815a7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">additionalModelOutput</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21AdditionalModelOutputRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21AdditionalModelOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK21AdditionalModelOutput"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK21AdditionalModelOutput"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__AdditionalModelOutputCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5d38254b21dfbad88b8bb02c16777875"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">additionalModelOutput</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21AdditionalModelOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization36deserializeExternalDraftTokensConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization36deserializeExternalDraftTokensConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization36deserializeExternalDraftTokensConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeExternalDraftTokensConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2bca807ceb86f58a385de9bdcc1bf481"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeExternalDraftTokensConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeExternalDraftTokensConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25ExternalDraftTokensConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK25ExternalDraftTokensConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK25ExternalDraftTokensConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ExternalDraftTokensConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1adbcd97e4e4d2822a1222fab34c3b3699"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25ExternalDraftTokensConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25ExternalDraftTokensConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK25ExternalDraftTokensConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK25ExternalDraftTokensConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ExternalDraftTokensConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a0c631d80dfe19e428b64e92e24ecbfc7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25ExternalDraftTokensConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializePromptTuningConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializePromptTuningConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializePromptTuningConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializePromptTuningConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9c20dc9bf0ec815a2fd91243e79f82a1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializePromptTuningConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializePromptTuningConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18PromptTuningConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18PromptTuningConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18PromptTuningConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__PromptTuningConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aeffa6dbca81617e4c8f2f151402de0aa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18PromptTuningConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18PromptTuningConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18PromptTuningConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18PromptTuningConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__PromptTuningConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a64114e901f6976ad2ede341a4ce46623"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18PromptTuningConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization26deserializeMultimodalInputERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization26deserializeMultimodalInputERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization26deserializeMultimodalInputERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeMultimodalInput__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac0e4bdab5f93ebfb2b738106cbc337c9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeMultimodalInput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializeMultimodalInputERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15MultimodalInputRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK15MultimodalInputRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK15MultimodalInputRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__MultimodalInputCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a4000bf5bdd80377efb5d22068e08f822"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">multimodalInput</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15MultimodalInputRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15MultimodalInput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK15MultimodalInput"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK15MultimodalInput"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__MultimodalInputCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aefd676a92e5574e4ac379ce5c074e053"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">multimodalInput</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15MultimodalInput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization22deserializeMropeConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization22deserializeMropeConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization22deserializeMropeConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeMropeConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab8a512a4577e8df9c91b4fab9bed3ed5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeMropeConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeMropeConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11MropeConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK11MropeConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK11MropeConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__MropeConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aec3a044c9bd6672dd486f6881843c67c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11MropeConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11MropeConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK11MropeConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK11MropeConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__MropeConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a887668fed866430fd21e2d0ea93f2c07"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11MropeConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization21deserializeLoraConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization21deserializeLoraConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization21deserializeLoraConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeLoraConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab6ae74552a93ff397c5af265c344fe56"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeLoraConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeLoraConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK10LoraConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK10LoraConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK10LoraConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__LoraConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aab0cf47956bc7bb1ee1452aa90edb6de"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK10LoraConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK10LoraConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK10LoraConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK10LoraConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__LoraConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a70a62ef2cdbc5a65dc3b61e4052a8133"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK10LoraConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization20deserializeCommStateERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization20deserializeCommStateERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization20deserializeCommStateERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeCommState__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a34443e07959170ea25fde6ef27452c97"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeCommState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeCommStateERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache9CommStateERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache9CommStateERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache9CommStateERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__kv_cache::CommStateCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a10777655cc16bcc53cd25de031bf04cd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache9CommStateERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache9CommStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache9CommStateE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache9CommStateE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__kv_cache::CommStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae3050aad60f9b26b95e5359353596359"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache9CommStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization22deserializeSocketStateERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization22deserializeSocketStateERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization22deserializeSocketStateERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSocketState__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2428a248a3d2e88853dca9f8fbd60d5a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSocketState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeSocketStateERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache11SocketStateERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache11SocketStateERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache11SocketStateERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__kv_cache::SocketStateCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8e412592cb9710e2d7a37df8a5decc9f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache11SocketStateERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache11SocketStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache11SocketStateE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache11SocketStateE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__kv_cache::SocketStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab85a5c8fd7ec4c2ff14cb51b738b71d1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache11SocketStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization21deserializeAgentStateERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization21deserializeAgentStateERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization21deserializeAgentStateERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeAgentState__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6757259d146f076e9d58600a8af3cee1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeAgentState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeAgentStateERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10AgentStateERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10AgentStateERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10AgentStateERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__kv_cache::AgentStateCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a09a40a9e6b52fc6a82c33cddbeac9e57"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10AgentStateERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10AgentStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10AgentStateE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10AgentStateE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__kv_cache::AgentStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a398a25ea47e1688bf59939c85a53d4ff"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10AgentStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization21deserializeCacheStateERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization21deserializeCacheStateERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization21deserializeCacheStateERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeCacheState__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a0703b669e35401e746cfa9a4ebe63ae2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeCacheState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeCacheStateERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10CacheStateERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10CacheStateERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10CacheStateERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__kv_cache::CacheStateCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1af1d1dfa6808bf6e306cfb816b1021f4c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10CacheStateERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10CacheStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10CacheStateE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10CacheStateE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__kv_cache::CacheStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aa2b26e9bdbcdc241a96a864ca2e6905a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10CacheStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDataTransceiverState__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac72aa5b4757d07a6178867c6c706e897"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDataTransceiverState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDataTransceiverState__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a80a4b33cf5225852f65b0ece648f77f7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDataTransceiverState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverStateRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverStateRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverStateRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DataTransceiverStateCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a15e862f6a3af7233b8b1171273421b8e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">dataTransceiverState</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverStateRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverState">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverState"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverState"></span><span id="tensorrt_llm::executor::Serialization::serialize__DataTransceiverStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a25862a57edf19782f80cdcab1b942386"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">dataTransceiverState</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20DataTransceiverState">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK20DataTransceiverState"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK20DataTransceiverState"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DataTransceiverStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6ca4f00139b3b74fe3638e9e0f4e33e1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">dataTransceiverState</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20DataTransceiverState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeContextPhaseParamsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeContextPhaseParamsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeContextPhaseParamsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeContextPhaseParams__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a94714fbc91d588c87fc573c0f08c710e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeContextPhaseParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeContextPhaseParamsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18ContextPhaseParamsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18ContextPhaseParamsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18ContextPhaseParamsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ContextPhaseParamsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aff42ad9117d0656ec2f7a1a9d30e5a16"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">contextPhaseParams</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18ContextPhaseParamsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18ContextPhaseParams"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ContextPhaseParamsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a31e8e60215fe54fc7ae438da792ffb7b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">contextPhaseParams</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization18deserializeRequestERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization18deserializeRequestERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization18deserializeRequestERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequest__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a52ea6afb73b932c514d8bf2fbacb5df2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequest</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization18deserializeRequestERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK7RequestRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK7RequestRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK7RequestRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a51b1f66893f945e1d48ce4f466ba1010"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">request</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK7RequestRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK7Request"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK7Request"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a15092605072969164a22559fcf61f6c1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">request</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization17deserializeTensorERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization17deserializeTensorERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization17deserializeTensorERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeTensor__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab32ab17b2c60fba0ee29097c2ea7e055"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeTensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeTensorERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6TensorRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK6TensorRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK6TensorRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__TensorCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8fe83cad52a85278ba6ff00c542a9214"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6TensorRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK6Tensor"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a85dbca4cdebadd45d7329329fcf656c0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization32deserializeSpecDecFastLogitsInfoERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization32deserializeSpecDecFastLogitsInfoERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization32deserializeSpecDecFastLogitsInfoERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSpecDecFastLogitsInfo__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aad3b65ee245884aa7b8984bd688be641"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE" title="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo"><span class="n"><span class="pre">SpeculativeDecodingFastLogitsInfo</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSpecDecFastLogitsInfo</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeSpecDecFastLogitsInfoERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK33SpeculativeDecodingFastLogitsInfoRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK33SpeculativeDecodingFastLogitsInfoRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK33SpeculativeDecodingFastLogitsInfoRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__SpeculativeDecodingFastLogitsInfoCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a973afba71c86101e4105c9c10f625714"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE" title="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo"><span class="n"><span class="pre">SpeculativeDecodingFastLogitsInfo</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">info</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK33SpeculativeDecodingFastLogitsInfoRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK33SpeculativeDecodingFastLogitsInfo">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK33SpeculativeDecodingFastLogitsInfo"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK33SpeculativeDecodingFastLogitsInfo"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__SpeculativeDecodingFastLogitsInfoCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a27e38a765aa5bd176bbbbfdd50489627"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE" title="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo"><span class="n"><span class="pre">SpeculativeDecodingFastLogitsInfo</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">info</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK33SpeculativeDecodingFastLogitsInfo" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization17deserializeResultERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization17deserializeResultERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization17deserializeResultERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeResult__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a258edbaa27d4bc82e5919f921aaff5b3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6ResultE" title="tensorrt_llm::executor::Result"><span class="n"><span class="pre">Result</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeResult</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeResultERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6ResultRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK6ResultRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK6ResultRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ResultCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae2bc066744d82a2457a974478a92d24d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6ResultE" title="tensorrt_llm::executor::Result"><span class="n"><span class="pre">Result</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">result</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6ResultRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Result">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK6Result"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK6Result"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ResultCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1acf908575acc37c7e106488e59f8aa4ff"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6ResultE" title="tensorrt_llm::executor::Result"><span class="n"><span class="pre">Result</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">result</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Result" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization27deserializeAdditionalOutputERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization27deserializeAdditionalOutputERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization27deserializeAdditionalOutputERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeAdditionalOutput__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2f8142cb0c1600970afbbd3938e92eb3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeAdditionalOutput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization27deserializeAdditionalOutputERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK16AdditionalOutputRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK16AdditionalOutputRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK16AdditionalOutputRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__AdditionalOutputCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1af1e903fc7019c49c6a2301f6f2495bb1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">additionalOutput</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK16AdditionalOutputRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK16AdditionalOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK16AdditionalOutput"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK16AdditionalOutput"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__AdditionalOutputCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a357d279378ee395c1ab95d4a761b5ad1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">additionalOutput</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK16AdditionalOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization19deserializeResponseERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization19deserializeResponseERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization19deserializeResponseERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeResponse__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae5a8b5e299fb2231fc41881b2fd12b31"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeResponse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization19deserializeResponseERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK8ResponseRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK8ResponseRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK8ResponseRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ResponseCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1dc5a101175bb9de92efa650e6d20e68"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">response</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK8ResponseRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK8Response">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK8Response"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK8Response"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ResponseCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1accb642acb602af27140cce717b5b18e8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">response</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK8Response" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization20deserializeResponsesERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization20deserializeResponsesERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization20deserializeResponsesERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeResponses__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a14d75bf69426bfa81113b5efe6d4a5cd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeResponsesERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI8ResponseEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI8ResponseEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI8ResponseEE"></span><span id="tensorrt_llm::executor::Serialization::serialize__std::vector:Response:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac95ee049bd2b4fc2d401e0596739df3d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">responses</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI8ResponseEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKvCacheConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization24deserializeKvCacheConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization24deserializeKvCacheConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKvCacheConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a93df88f47d0cbfa681af463ed1b680d8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKvCacheConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKvCacheConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK13KvCacheConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK13KvCacheConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK13KvCacheConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KvCacheConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8e27bfce7f4b56448cdb1e40596ad5b6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK13KvCacheConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK13KvCacheConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK13KvCacheConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK13KvCacheConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KvCacheConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad457e1d524480c49cc90ac55aeda1943"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK13KvCacheConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeDynamicBatchConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeDynamicBatchConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeDynamicBatchConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDynamicBatchConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ade33854bf404f5410198d2b54e33f6c0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDynamicBatchConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeDynamicBatchConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18DynamicBatchConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18DynamicBatchConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18DynamicBatchConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DynamicBatchConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab8c488d557304327ad8704a319fb807a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">dynamicBatchConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18DynamicBatchConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18DynamicBatchConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18DynamicBatchConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18DynamicBatchConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DynamicBatchConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad0dbc1675cda40c8096a0356791f4c19"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">dynamicBatchConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18DynamicBatchConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization26deserializeSchedulerConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization26deserializeSchedulerConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization26deserializeSchedulerConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSchedulerConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5721cef6b72c2a079b5dbe94a7f9249e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSchedulerConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializeSchedulerConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15SchedulerConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK15SchedulerConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK15SchedulerConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__SchedulerConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a75d2e0606543f60ed2bf6bbaf00456f4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">schedulerConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15SchedulerConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15SchedulerConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK15SchedulerConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK15SchedulerConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__SchedulerConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1af3642582eaeb939ba265089c2fee753a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">schedulerConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15SchedulerConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization40deserializeExtendedRuntimePerfKnobConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization40deserializeExtendedRuntimePerfKnobConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization40deserializeExtendedRuntimePerfKnobConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeExtendedRuntimePerfKnobConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac507a4af186242ca004a666769d156d1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeExtendedRuntimePerfKnobConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization40deserializeExtendedRuntimePerfKnobConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK29ExtendedRuntimePerfKnobConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK29ExtendedRuntimePerfKnobConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK29ExtendedRuntimePerfKnobConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ExtendedRuntimePerfKnobConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5b53e64d28f954ae520fba4d4395b070"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">extendedRuntimePerfKnobConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK29ExtendedRuntimePerfKnobConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK29ExtendedRuntimePerfKnobConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK29ExtendedRuntimePerfKnobConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK29ExtendedRuntimePerfKnobConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ExtendedRuntimePerfKnobConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9f2fe7ee62a55bf3a2b033e4a9556674"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">extendedRuntimePerfKnobConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK29ExtendedRuntimePerfKnobConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeParallelConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeParallelConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeParallelConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeParallelConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8ddf1b2798dbf98d3eac0759c4646360"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeParallelConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeParallelConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ParallelConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14ParallelConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14ParallelConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ParallelConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a352018daa6065ab5d7f8d40df594f790"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">parallelConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ParallelConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ParallelConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK14ParallelConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK14ParallelConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ParallelConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a903b5bc4b669a8e8cfdff9206512969a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">parallelConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ParallelConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization26deserializePeftCacheConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization26deserializePeftCacheConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization26deserializePeftCacheConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializePeftCacheConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a36ab38bb0f5fe55b9829c8177c93e91f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializePeftCacheConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializePeftCacheConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15PeftCacheConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK15PeftCacheConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK15PeftCacheConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__PeftCacheConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a28c920c3cbb2c7f97678a0ed2fd704cb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">peftCacheConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15PeftCacheConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15PeftCacheConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK15PeftCacheConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK15PeftCacheConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__PeftCacheConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae6b46926ad9c8754e7cd89c1b9c8bd36"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">peftCacheConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15PeftCacheConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeOrchestratorConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeOrchestratorConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeOrchestratorConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeOrchestratorConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae025cd58097d96dc74cb1acd6207325d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeOrchestratorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeOrchestratorConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18OrchestratorConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18OrchestratorConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18OrchestratorConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__OrchestratorConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a7864913d0c2362e972654c2a5613b566"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">orchestratorConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18OrchestratorConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18OrchestratorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18OrchestratorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18OrchestratorConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__OrchestratorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a255cb93a98ccdf748db37f8b492520d3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">orchestratorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18OrchestratorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeDecodingModeERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeDecodingModeERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeDecodingModeERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDecodingMode__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6793f998581467d3540e2ed35167be15"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDecodingMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeDecodingModeERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12DecodingModeRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12DecodingModeRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12DecodingModeRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DecodingModeCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a0b3f4dd50c3174e5522cd945a55416a8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decodingMode</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12DecodingModeRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12DecodingMode">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12DecodingMode"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12DecodingMode"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DecodingModeCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a68100e2eb882d29677a4c9ed45cfd7a8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decodingMode</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12DecodingMode" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization34deserializeLookaheadDecodingConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization34deserializeLookaheadDecodingConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization34deserializeLookaheadDecodingConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeLookaheadDecodingConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a09e3d04d4394b505c41a2f88d2395de2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeLookaheadDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization34deserializeLookaheadDecodingConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK23LookaheadDecodingConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK23LookaheadDecodingConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK23LookaheadDecodingConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__LookaheadDecodingConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a7f7d5716bc415282ad7d870cee8a0af2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lookaheadDecodingConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK23LookaheadDecodingConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK23LookaheadDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK23LookaheadDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK23LookaheadDecodingConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__LookaheadDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5dce85091047f4b569a106630ddd0840"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lookaheadDecodingConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK23LookaheadDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization22deserializeEagleConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization22deserializeEagleConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization22deserializeEagleConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeEagleConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1abfe63c04a67358325f7de9c1a84bd5d9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeEagleConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeEagleConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11EagleConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK11EagleConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK11EagleConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__EagleConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a37588c9152027aec9b1470c1aab5cca7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">eagleConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11EagleConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11EagleConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK11EagleConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK11EagleConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__EagleConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a0c47859bf5f406af776bca0e1dc91a83"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">eagleConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11EagleConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization36deserializeSpeculativeDecodingConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization36deserializeSpeculativeDecodingConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization36deserializeSpeculativeDecodingConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSpeculativeDecodingConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9715efdef8cfea9f207849268600fa00"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSpeculativeDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeSpeculativeDecodingConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25SpeculativeDecodingConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK25SpeculativeDecodingConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK25SpeculativeDecodingConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__SpeculativeDecodingConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae22a12c73325733708048218506f7ee9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">specDecConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25SpeculativeDecodingConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25SpeculativeDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK25SpeculativeDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK25SpeculativeDecodingConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__SpeculativeDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9c03fac4af9f93b327425530008079ae"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">specDecConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25SpeculativeDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeGuidedDecodingConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8a1d553784a718951697122d7f0f83a5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeGuidedDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__GuidedDecodingConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a070aaf03c054306d09b32a63a72e22ca"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__GuidedDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6df9f55c33fc9bfbb0237293824af7ca"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingParamsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingParamsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingParamsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeGuidedDecodingParams__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1afa984172377b9e1e3c65fcf107178d44"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeGuidedDecodingParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingParamsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingParamsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingParamsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingParamsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__GuidedDecodingParamsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1abaa14972d252807ca10179e9d836c3dc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingParams</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingParamsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingParams"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingParams"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__GuidedDecodingParamsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a48b0a0db79fd6aa7cc5dda8f4cc91dc4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingParams</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKvCacheRetentionConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization33deserializeKvCacheRetentionConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization33deserializeKvCacheRetentionConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKvCacheRetentionConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae10d1c31c6521f41ec81003c1a186baa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKvCacheRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKvCacheRetentionConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KvCacheRetentionConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK22KvCacheRetentionConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK22KvCacheRetentionConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KvCacheRetentionConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2ce73676e73612b6c129a1c22550ef91"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheRetentionConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KvCacheRetentionConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KvCacheRetentionConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK22KvCacheRetentionConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK22KvCacheRetentionConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KvCacheRetentionConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a26cbc2d54e6ba3c1e68c52e6943810f1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheRetentionConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KvCacheRetentionConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization36deserializeTokenRangeRetentionConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization36deserializeTokenRangeRetentionConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization36deserializeTokenRangeRetentionConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeTokenRangeRetentionConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a915e5c3142c7fc92365b4d58bef33596"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeTokenRangeRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeTokenRangeRetentionConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KvCacheRetentionConfig::TokenRangeRetentionConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2e2917d1129a25aa6e6772c13c0e269a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tokenRangeRetentionConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KvCacheRetentionConfig::TokenRangeRetentionConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aa5e510b891a5e3b45e5f6d0cb2af3176"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tokenRangeRetentionConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeDecodingConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeDecodingConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeDecodingConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDecodingConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a60e192f644c0e8693f0a3b12d6bb60e0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDecodingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeDecodingConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14DecodingConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14DecodingConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14DecodingConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DecodingConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad1a7a61b52c5d823406a5a2bb05edb21"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decodingConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14DecodingConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14DecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK14DecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK14DecodingConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8b8d2707b981fbb788a24a31c1170a72"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decodingConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14DecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization22deserializeDebugConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization22deserializeDebugConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization22deserializeDebugConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDebugConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9308aeea31b9888f81b340b19772bf10"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDebugConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeDebugConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11DebugConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK11DebugConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK11DebugConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DebugConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1afa49754c743c041ae7c7cdace53bed38"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">debugConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11DebugConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11DebugConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK11DebugConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK11DebugConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DebugConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aa75fc302341a6adb991d635ef4e2ba0b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">debugConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11DebugConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization33deserializeCacheTransceiverConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization33deserializeCacheTransceiverConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization33deserializeCacheTransceiverConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeCacheTransceiverConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5e68b0e48306b645ffe725bc716eec09"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeCacheTransceiverConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeCacheTransceiverConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22CacheTransceiverConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK22CacheTransceiverConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK22CacheTransceiverConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__CacheTransceiverConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a7a6423a86e7fb79045684c7f8774c3ba"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">cacheTransceiverConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22CacheTransceiverConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22CacheTransceiverConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK22CacheTransceiverConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK22CacheTransceiverConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__CacheTransceiverConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a85e2a5315ec346704a13a9c36ddce7d8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">cacheTransceiverConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22CacheTransceiverConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeExecutorConfigERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeExecutorConfigERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeExecutorConfigERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeExecutorConfig__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a3bdc71d1f61ffbe8192eec4c69a6f863"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeExecutorConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeExecutorConfigERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ExecutorConfigRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14ExecutorConfigRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14ExecutorConfigRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__ExecutorConfigCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8d327c320f459d69fd6561a420558674"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ExecutorConfigRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aeabfd8c6625ad85aed6ef2cb72f8cd66"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKvCacheStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeKvCacheStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeKvCacheStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKvCacheStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a00364ba63856dbd99f89ae1e45770ffa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="tensorrt_llm::executor::KvCacheStats"><span class="n"><span class="pre">KvCacheStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKvCacheStats</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKvCacheStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KvCacheStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12KvCacheStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12KvCacheStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KvCacheStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a92003eb85d2979660e29056b4f81cf0a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="tensorrt_llm::executor::KvCacheStats"><span class="n"><span class="pre">KvCacheStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheStats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KvCacheStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KvCacheStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12KvCacheStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12KvCacheStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KvCacheStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5e3c1533994cd82ce444b0c632bb6fbe"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="tensorrt_llm::executor::KvCacheStats"><span class="n"><span class="pre">KvCacheStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheStats</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KvCacheStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization30deserializeStaticBatchingStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization30deserializeStaticBatchingStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization30deserializeStaticBatchingStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeStaticBatchingStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad66b899cda69588f655b08dd36f15925"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE" title="tensorrt_llm::executor::StaticBatchingStats"><span class="n"><span class="pre">StaticBatchingStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeStaticBatchingStats</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization30deserializeStaticBatchingStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK19StaticBatchingStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK19StaticBatchingStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK19StaticBatchingStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__StaticBatchingStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1187b7a88a0e5deb38ff8e593372e984"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE" title="tensorrt_llm::executor::StaticBatchingStats"><span class="n"><span class="pre">StaticBatchingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">staticBatchingStats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK19StaticBatchingStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK19StaticBatchingStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK19StaticBatchingStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK19StaticBatchingStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__StaticBatchingStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6fe6208a4f78db241e6d9c6b2afd24e5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE" title="tensorrt_llm::executor::StaticBatchingStats"><span class="n"><span class="pre">StaticBatchingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">staticBatchingStats</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK19StaticBatchingStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization32deserializeInflightBatchingStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization32deserializeInflightBatchingStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization32deserializeInflightBatchingStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeInflightBatchingStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6ddddf159c364fc1af5d7ba7a1a5c9cf"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE" title="tensorrt_llm::executor::InflightBatchingStats"><span class="n"><span class="pre">InflightBatchingStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeInflightBatchingStats</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeInflightBatchingStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21InflightBatchingStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK21InflightBatchingStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK21InflightBatchingStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__InflightBatchingStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1e571817bc0585695894442bd4697fc6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE" title="tensorrt_llm::executor::InflightBatchingStats"><span class="n"><span class="pre">InflightBatchingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">inflightBatchingStats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21InflightBatchingStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21InflightBatchingStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK21InflightBatchingStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK21InflightBatchingStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__InflightBatchingStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1adfd862d5d685d90dfbec4e742c340cd7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE" title="tensorrt_llm::executor::InflightBatchingStats"><span class="n"><span class="pre">InflightBatchingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">inflightBatchingStats</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21InflightBatchingStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization28deserializeSpecDecodingStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization28deserializeSpecDecodingStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization28deserializeSpecDecodingStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeSpecDecodingStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a08c01ef4092ee77ba37d30a56e7a567c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE" title="tensorrt_llm::executor::SpecDecodingStats"><span class="n"><span class="pre">SpecDecodingStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeSpecDecodingStats</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeSpecDecodingStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17SpecDecodingStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK17SpecDecodingStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK17SpecDecodingStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__SpecDecodingStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aec84ae79883f50f60dfd65e16031cb39"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE" title="tensorrt_llm::executor::SpecDecodingStats"><span class="n"><span class="pre">SpecDecodingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">specDecodingStats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17SpecDecodingStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17SpecDecodingStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK17SpecDecodingStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK17SpecDecodingStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__SpecDecodingStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1afc687fe2efc5c0d3dca4b056c3d2f240"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE" title="tensorrt_llm::executor::SpecDecodingStats"><span class="n"><span class="pre">SpecDecodingStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">specDecodingStats</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17SpecDecodingStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeIterationStats__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9797c0d1af10c396b36f548de7d2e8e2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeIterationStats</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeIterationStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad1bbfdc68d6c6d0a8342317fac32bd93"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeIterationStats</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14IterationStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14IterationStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__IterationStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2ed4d91cf95dfd30083fa0154f8d6243"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">iterStats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK14IterationStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK14IterationStats"></span><span id="tensorrt_llm::executor::Serialization::serialize__IterationStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6950808f968c435d4efa2696421e49be"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">iterStats</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14IterationStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK14IterationStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK14IterationStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__IterationStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a24a30c0a00c745499a73d7f754c9e67a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">iterStats</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14IterationStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI14IterationStatsEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI14IterationStatsEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI14IterationStatsEE"></span><span id="tensorrt_llm::executor::Serialization::serialize__std::vector:IterationStats:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a18501fffc9db30dbb1def7281d639d04"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">iterStatsVec</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI14IterationStatsEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization28deserializeIterationStatsVecERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization28deserializeIterationStatsVecERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization28deserializeIterationStatsVecERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeIterationStatsVec__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad25c05a8bcff2152ffae00bcfd3c6553"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeIterationStatsVec</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeIterationStatsVecERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization33deserializeDisServingRequestStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization33deserializeDisServingRequestStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization33deserializeDisServingRequestStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeDisServingRequestStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ae3f88de690b770067d41e9de565365da"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE" title="tensorrt_llm::executor::DisServingRequestStats"><span class="n"><span class="pre">DisServingRequestStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeDisServingRequestStats</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeDisServingRequestStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22DisServingRequestStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK22DisServingRequestStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK22DisServingRequestStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__DisServingRequestStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a4b68bd59f29204220da5cf065f98fc10"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE" title="tensorrt_llm::executor::DisServingRequestStats"><span class="n"><span class="pre">DisServingRequestStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">stats</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22DisServingRequestStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22DisServingRequestStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK22DisServingRequestStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK22DisServingRequestStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__DisServingRequestStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad55ef9f5fa3d7225411143f51d3e7297"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE" title="tensorrt_llm::executor::DisServingRequestStats"><span class="n"><span class="pre">DisServingRequestStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">disServingRequestStats</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22DisServingRequestStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStageERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeRequestStageERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeRequestStageERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestStage__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1abb6dd3b8c12d6d3a9921e39749bbdcda"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE" title="tensorrt_llm::executor::RequestStage"><span class="n"><span class="pre">RequestStage</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestStage</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStageERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStageRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12RequestStageRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12RequestStageRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestStageCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a696465e544a9f4a6377d17ab029d47dc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE" title="tensorrt_llm::executor::RequestStage"><span class="n"><span class="pre">RequestStage</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestStage</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStageRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStage">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStage"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStage"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestStageCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1acdd5dd282522b9bb0ff940bd2c7c1d86"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE" title="tensorrt_llm::executor::RequestStage"><span class="n"><span class="pre">RequestStage</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestStage</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStage" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStatsERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeRequestStatsERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeRequestStatsERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestStats__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a36d0b08bddac8c6c9252effde1127b86"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="tensorrt_llm::executor::RequestStats"><span class="n"><span class="pre">RequestStats</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestStats</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStatsERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStatsRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12RequestStatsRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12RequestStatsRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestStatsCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a64b2978d2361e3d5c31a72f82ea99c30"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="tensorrt_llm::executor::RequestStats"><span class="n"><span class="pre">RequestStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStatsRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStats"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStats"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aed0abb450ac08a383e844ff44fb721d9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="tensorrt_llm::executor::RequestStats"><span class="n"><span class="pre">RequestStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestStatsPerIteration__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a44549f08640743ae8609e2d9aea5f8aa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestStatsPerIteration</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestStatsPerIteration__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a490b8d2bef2ca33876374bf40ea54588"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestStatsPerIteration</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIterationRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIterationRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIterationRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestStatsPerIterationCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a76bea696802f2256018e9ae72c30c9a4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIterationRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIteration">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIteration"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIteration"></span><span id="tensorrt_llm::executor::Serialization::serialize__RequestStatsPerIterationCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac6b4c8f3e213096649b7c76d85911231"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIteration" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK24RequestStatsPerIteration">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK24RequestStatsPerIteration"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK24RequestStatsPerIteration"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__RequestStatsPerIterationCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5ab0e83eb6c825653558f13aa3730480"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK24RequestStatsPerIteration" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI24RequestStatsPerIterationEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI24RequestStatsPerIterationEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI24RequestStatsPerIterationEE"></span><span id="tensorrt_llm::executor::Serialization::serialize__std::vector:RequestStatsPerIteration:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5ff3fdcfdc4c9c9cca1716c0efdbf04b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestStatsVec</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI24RequestStatsPerIterationEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization38deserializeRequestStatsPerIterationVecERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization38deserializeRequestStatsPerIterationVecERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization38deserializeRequestStatsPerIterationVecERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeRequestStatsPerIterationVec__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a215b502eda1023303c092284649e5e0b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeRequestStatsPerIterationVec</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization38deserializeRequestStatsPerIterationVecERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt5dequeI12KVCacheEventEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKNSt5dequeI12KVCacheEventEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKNSt5dequeI12KVCacheEventEE"></span><span id="tensorrt_llm::executor::Serialization::serialize__std::deque:KVCacheEvent:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a15f594bbdf360aee1dfa94085159decd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheEvents</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt5dequeI12KVCacheEventEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKVCacheEventsERNSt6vectorIcEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization24deserializeKVCacheEventsERNSt6vectorIcEE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization24deserializeKVCacheEventsERNSt6vectorIcEE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheEvents__std::vector:c:R"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a4fda29552c786825c9be0243f1e9255d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheEvents</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKVCacheEventsERNSt6vectorIcEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KVCacheEvent">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK12KVCacheEvent"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK12KVCacheEvent"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheEventCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad350db68f3f0ce2311e77900dcddc753"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KVCacheEvent" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KVCacheEventRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK12KVCacheEventRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK12KVCacheEventRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheEventCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a3c009a3309bee3a5149ad9f58e599161"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">event</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KVCacheEventRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKVCacheEventERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization23deserializeKVCacheEventERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization23deserializeKVCacheEventERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheEvent__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a6628226668ad35df282d72a07a9e558a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheEvent</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKVCacheEventERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheCreatedData">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheCreatedData"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheCreatedData"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheCreatedDataCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a5d3454e960734276cf070f4dbaeec5db"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE" title="tensorrt_llm::executor::KVCacheCreatedData"><span class="n"><span class="pre">KVCacheCreatedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheCreatedData" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheCreatedDataRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheCreatedDataRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheCreatedDataRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheCreatedDataCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad191970a43a3993bdbf257cf9cf58f18"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE" title="tensorrt_llm::executor::KVCacheCreatedData"><span class="n"><span class="pre">KVCacheCreatedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheCreatedDataRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheCreatedDataERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeKVCacheCreatedDataERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeKVCacheCreatedDataERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheCreatedData__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a0aab5cd525d06c72c083e9a4af012ebd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE" title="tensorrt_llm::executor::KVCacheCreatedData"><span class="n"><span class="pre">KVCacheCreatedData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheCreatedData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheCreatedDataERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17KVCacheStoredData">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK17KVCacheStoredData"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK17KVCacheStoredData"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheStoredDataCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab9c88475928773b4d7f1b6a4753dfcaf"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE" title="tensorrt_llm::executor::KVCacheStoredData"><span class="n"><span class="pre">KVCacheStoredData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17KVCacheStoredData" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17KVCacheStoredDataRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK17KVCacheStoredDataRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK17KVCacheStoredDataRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheStoredDataCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1abdb80ea0ec8469e8c5d8ed5698f65cef"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE" title="tensorrt_llm::executor::KVCacheStoredData"><span class="n"><span class="pre">KVCacheStoredData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17KVCacheStoredDataRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization28deserializeKVCacheStoredDataERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization28deserializeKVCacheStoredDataERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization28deserializeKVCacheStoredDataERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheStoredData__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1afcde92fa40b70b0283bc16f5b8866de2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE" title="tensorrt_llm::executor::KVCacheStoredData"><span class="n"><span class="pre">KVCacheStoredData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheStoredData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeKVCacheStoredDataERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KVCacheStoredBlockData">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK22KVCacheStoredBlockData"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK22KVCacheStoredBlockData"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheStoredBlockDataCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a117bc4ccf3a16718a8f7d1e9b46f034e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE" title="tensorrt_llm::executor::KVCacheStoredBlockData"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KVCacheStoredBlockData" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KVCacheStoredBlockDataRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK22KVCacheStoredBlockDataRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK22KVCacheStoredBlockDataRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheStoredBlockDataCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ab5e207b53ddde588d5025b48a887e9ba"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE" title="tensorrt_llm::executor::KVCacheStoredBlockData"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KVCacheStoredBlockDataRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKVCacheStoredBlockDataERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization33deserializeKVCacheStoredBlockDataERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization33deserializeKVCacheStoredBlockDataERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheStoredBlockData__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a8b8d280d4b2bf244beba889083789963"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE" title="tensorrt_llm::executor::KVCacheStoredBlockData"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheStoredBlockData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKVCacheStoredBlockDataERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheRemovedData">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheRemovedData"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheRemovedData"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheRemovedDataCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1eef8a8283639b8ab5399771fad5cbaa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE" title="tensorrt_llm::executor::KVCacheRemovedData"><span class="n"><span class="pre">KVCacheRemovedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheRemovedData" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheRemovedDataRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheRemovedDataRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheRemovedDataRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheRemovedDataCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a1713bb88a7eb92026a21145ea39028ed"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE" title="tensorrt_llm::executor::KVCacheRemovedData"><span class="n"><span class="pre">KVCacheRemovedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheRemovedDataRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheRemovedDataERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeKVCacheRemovedDataERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeKVCacheRemovedDataERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheRemovedData__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a34130637eabdf2ce4b3ac85b55808604"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE" title="tensorrt_llm::executor::KVCacheRemovedData"><span class="n"><span class="pre">KVCacheRemovedData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheRemovedData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheRemovedDataERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor13Serialization14serializedSizeE6size_tRK16KVCacheEventDiffI1TE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor13Serialization14serializedSizeERK16KVCacheEventDiffI1TE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor13Serialization14serializedSizeERK16KVCacheEventDiffI1TE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a18885ec3adbd2123319f3db8539465b6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization14serializedSizeE6size_tRK16KVCacheEventDiffI1TE" title="tensorrt_llm::executor::Serialization::serializedSize::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization14serializedSizeE6size_tRK16KVCacheEventDiffI1TE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor13Serialization9serializeEvRK16KVCacheEventDiffI1TERNSt7ostreamE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor13Serialization9serializeERK16KVCacheEventDiffI1TERNSt7ostreamE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor13Serialization9serializeERK16KVCacheEventDiffI1TERNSt7ostreamE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ad147a1556f8a8a6c2e56f1fe7276e1f1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization9serializeEvRK16KVCacheEventDiffI1TERNSt7ostreamE" title="tensorrt_llm::executor::Serialization::serialize::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization9serializeEvRK16KVCacheEventDiffI1TERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffE16KVCacheEventDiffI1TERNSt7istreamE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffERNSt7istreamE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffERNSt7istreamE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aa6dafad1463f24a6c762cba6e07d7369"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffE16KVCacheEventDiffI1TERNSt7istreamE" title="tensorrt_llm::executor::Serialization::deserializeKVCacheEventDiff::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheEventDiff</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffE16KVCacheEventDiffI1TERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheUpdatedData">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheUpdatedData"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheUpdatedData"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__KVCacheUpdatedDataCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a2ecd12b0c4120163c18422ccdf8aa62d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheUpdatedData" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheUpdatedDataRNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheUpdatedDataRNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheUpdatedDataRNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__KVCacheUpdatedDataCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a7b9feacd7c16309420b5c08d421f0a95"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheUpdatedDataRNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheUpdatedDataERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization29deserializeKVCacheUpdatedDataERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization29deserializeKVCacheUpdatedDataERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeKVCacheUpdatedData__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aeab35ee48ee28c4f9a6dc43e082ee0c6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeKVCacheUpdatedData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheUpdatedDataERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm7runtime11UniqueTokenE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm7runtime11UniqueTokenE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm7runtime11UniqueTokenE"></span><span id="tensorrt_llm::executor::Serialization::serializedSize__tensorrt_llm::runtime::UniqueTokenCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a81b6ef1d34047d8cf15ad73e34e2e8dc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serializedSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="tensorrt_llm::runtime::UniqueToken"><span class="n"><span class="pre">UniqueToken</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">token</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm7runtime11UniqueTokenE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm7runtime11UniqueTokenERNSt7ostreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm7runtime11UniqueTokenERNSt7ostreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm7runtime11UniqueTokenERNSt7ostreamE"></span><span id="tensorrt_llm::executor::Serialization::serialize__tensorrt_llm::runtime::UniqueTokenCR.osR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1aed121944fed45102774c4b4e64145134"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">serialize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="tensorrt_llm::runtime::UniqueToken"><span class="n"><span class="pre">UniqueToken</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">token</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm7runtime11UniqueTokenERNSt7ostreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization22deserializeUniqueTokenERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization22deserializeUniqueTokenERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization22deserializeUniqueTokenERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeUniqueToken__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a03c73886391c9348ee1929d99866b9dc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="tensorrt_llm::runtime::UniqueToken"><span class="n"><span class="pre">UniqueToken</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeUniqueToken</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeUniqueTokenERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization17deserializeStringERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization17deserializeStringERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization17deserializeStringERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeString__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1ac520ac5f4cc9ec6f5b6bf53ab5b7da8f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeString</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeStringERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization15deserializeBoolERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization15deserializeBoolERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization15deserializeBoolERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeBool__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a19d98b5105004255595bfeecd7cecb4a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeBool</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization15deserializeBoolERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13Serialization20deserializeModelTypeERNSt7istreamE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13Serialization20deserializeModelTypeERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm8executor13Serialization20deserializeModelTypeERNSt7istreamE"></span><span id="tensorrt_llm::executor::Serialization::deserializeModelType__isR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Serialization_1a9944e16a2c2b64e441b36fa72afb2e6b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="tensorrt_llm::executor::ModelType"><span class="n"><span class="pre">ModelType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deserializeModelType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">is</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeModelTypeERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor_1_1kv__cache"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kv_cache</span></span></span><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="types-h">
|
||
<h2>types.h<a class="headerlink" href="#types-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9TensorPtrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm8executor9TensorPtrE"></span><span class="target" id="types_8h_1a32a3846eb7d506ec2f4699f052f54dda"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9TensorPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10SizeType32E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm8executor10SizeType32E"></span><span class="target" id="types_8h_1ad818c2e487265ea3ec0ddd760b768085"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10SizeType64E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10SizeType64E"></span><span id="_CPPv2N12tensorrt_llm8executor10SizeType64E"></span><span class="target" id="types_8h_1acda8a22d5fd4b8f6f92ce04c779cf088"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9FloatTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9FloatTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor9FloatTypeE"></span><span class="target" id="types_8h_1a48053cc72a5a67b3c19c817fb963ecea"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FloatType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="kt"><span class="pre">float</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11TokenIdTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11TokenIdTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor11TokenIdTypeE"></span><span class="target" id="types_8h_1a5658d78655723ba6fac682b945cc6788"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TokenIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9VecTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9VecTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor9VecTokensE"></span><span class="target" id="types_8h_1a3f6e6a65b5e81747e30820e27d184e91"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">VecTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10BeamTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10BeamTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor10BeamTokensE"></span><span class="target" id="types_8h_1afc3b526e44121eda1f3344e1d611f688"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BeamTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10BeamTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6IdTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6IdTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor6IdTypeE"></span><span class="target" id="types_8h_1ab9563a6f39b5785365973b84532f9353"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16VecTokenExtraIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor16VecTokenExtraIdsE"></span><span class="target" id="types_8h_1a3075281db711a71a376ec24fe11dd2c7"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">VecTokenExtraIds</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13IterationTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13IterationTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor13IterationTypeE"></span><span class="target" id="types_8h_1ad5297846b0ec6db8536e14e970e0e09c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IterationType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14RandomSeedTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor14RandomSeedTypeE"></span><span class="target" id="types_8h_1aa7776f1267bf68fcfd4228fb0cc38bfc"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RandomSeedType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11VecLogProbsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11VecLogProbsE"></span><span id="_CPPv2N12tensorrt_llm8executor11VecLogProbsE"></span><span class="target" id="types_8h_1af438a899c644096dfd518bee78fad2dd"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">VecLogProbs</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11VecLogProbsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9StreamPtrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9StreamPtrE"></span><span id="_CPPv2N12tensorrt_llm8executor9StreamPtrE"></span><span class="target" id="types_8h_1a37eb662d6bef7e3702dac46671c3f543"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9StreamPtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16MillisecondsTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor16MillisecondsTypeE"></span><span class="target" id="types_8h_1a45b73aeba0f7c11070630d46720bfe32"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MillisecondsType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15CacheSaltIDTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor15CacheSaltIDTypeE"></span><span class="target" id="types_8h_1a738fff2a4a4cf339d657bb970285efdb"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheSaltIDType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19LogitsPostProcessorE"></span><span id="_CPPv2N12tensorrt_llm8executor19LogitsPostProcessorE"></span><span class="target" id="types_8h_1a2ec193a9bb683333faee35fb0a801320"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LogitsPostProcessor</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">function</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BeamTokensE" title="tensorrt_llm::executor::BeamTokens"><span class="n"><span class="pre">BeamTokens</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9StreamPtrE" title="tensorrt_llm::executor::StreamPtr"><span class="n"><span class="pre">StreamPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22LogitsPostProcessorMapE"></span><span id="_CPPv2N12tensorrt_llm8executor22LogitsPostProcessorMapE"></span><span class="target" id="types_8h_1ad609221eb06a044bdfa9afc31ac4b69b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LogitsPostProcessorMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE" title="tensorrt_llm::executor::LogitsPostProcessor"><span class="n"><span class="pre">LogitsPostProcessor</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor26LogitsPostProcessorBatchedE"></span><span id="_CPPv2N12tensorrt_llm8executor26LogitsPostProcessorBatchedE"></span><span class="target" id="types_8h_1a0a7b5ffd7a5228373ede89d57bf68236"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LogitsPostProcessorBatched</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">function</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">(</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">reference_wrapper</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BeamTokensE" title="tensorrt_llm::executor::BeamTokens"><span class="n"><span class="pre">BeamTokens</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9StreamPtrE" title="tensorrt_llm::executor::StreamPtr"><span class="n"><span class="pre">StreamPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13MedusaChoicesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13MedusaChoicesE"></span><span id="_CPPv2N12tensorrt_llm8executor13MedusaChoicesE"></span><span class="target" id="types_8h_1a6ee0e4eac427f8b44cd02cad361df400"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaChoices</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12EagleChoicesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12EagleChoicesE"></span><span id="_CPPv2N12tensorrt_llm8executor12EagleChoicesE"></span><span class="target" id="types_8h_1a79d9befd7352757e61700637c1a2fe84"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleChoices</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12PriorityTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12PriorityTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor12PriorityTypeE"></span><span class="target" id="types_8h_1a80b67583441a85102ace282b7e2f72af"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PriorityType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="kt"><span class="pre">float</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10BufferViewE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10BufferViewE"></span><span id="_CPPv2N12tensorrt_llm8executor10BufferViewE"></span><span class="target" id="types_8h_1a4f7ffe88ce0e39d4b8e29be9384975c7"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferView</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">basic_string_view</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">uint8_t</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10BufferViewE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-enums">Enums</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8DataTypeE"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96e"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType5kBOOLE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType5kBOOLE"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType5kBOOLE"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea65d294ade613161e5ea557e935133213"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBOOL</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType5kBOOLE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType6kUINT8E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType6kUINT8E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType6kUINT8E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea159244e0ab6cea87daa7eaa8f6e19937"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUINT8</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType6kUINT8E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType5kINT8E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType5kINT8E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType5kINT8E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea69c1a4a69db0e50820cf63122f90ad09"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kINT8</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType5kINT8E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType6kINT32E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType6kINT32E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType6kINT32E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96eabd073fcbb15020b25a70e2cd95f9f4a9"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kINT32</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT32E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType6kINT64E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType6kINT64E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType6kINT64E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea223b86006bb063f2a58200621e0656d5"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kINT64</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT64E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType5kBF16E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType5kBF16E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType5kBF16E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea7b2caf2ed7b5f4f177fe3207cd198791"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBF16</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType5kBF16E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType4kFP8E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType4kFP8E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType4kFP8E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea7397615c6bee5b62289fc7cceb82fbf7"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kFP8</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType4kFP8E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType5kFP16E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType5kFP16E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType5kFP16E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea56e4ef5e47a48568bd24c4e0aaabcead"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kFP16</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP16E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType5kFP32E">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType5kFP32E"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType5kFP32E"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea6c6463a7e81555667de0d7b49101a701"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kFP32</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP32E" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8DataType8kUNKNOWNE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8DataType8kUNKNOWNE"></span><span id="_CPPv2N12tensorrt_llm8executor8DataType8kUNKNOWNE"></span><span class="target" id="types_8h_1ad496a8556f8507f89bea4b8b7a7fc96ea16bb40b9102367393ecf8213078d1c6e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUNKNOWN</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8DataType8kUNKNOWNE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11RequestTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11RequestTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor11RequestTypeE"></span><span class="target" id="types_8h_1a2fa0faddbb57562b0cb8008594af7e39"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE"></span><span id="_CPPv2N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE"></span><span class="target" id="types_8h_1a2fa0faddbb57562b0cb8008594af7e39a99e4eb3c524741c99350cd470c463547"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">REQUEST_TYPE_CONTEXT_AND_GENERATION</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11RequestType25REQUEST_TYPE_CONTEXT_ONLYE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11RequestType25REQUEST_TYPE_CONTEXT_ONLYE"></span><span id="_CPPv2N12tensorrt_llm8executor11RequestType25REQUEST_TYPE_CONTEXT_ONLYE"></span><span class="target" id="types_8h_1a2fa0faddbb57562b0cb8008594af7e39a2f1a58e3c83a5e91847f15370f6493e2"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">REQUEST_TYPE_CONTEXT_ONLY</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11RequestType25REQUEST_TYPE_CONTEXT_ONLYE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11RequestType28REQUEST_TYPE_GENERATION_ONLYE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11RequestType28REQUEST_TYPE_GENERATION_ONLYE"></span><span id="_CPPv2N12tensorrt_llm8executor11RequestType28REQUEST_TYPE_GENERATION_ONLYE"></span><span class="target" id="types_8h_1a2fa0faddbb57562b0cb8008594af7e39a71b2d6a7108f89c26847d4c49a1ac0a1"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">REQUEST_TYPE_GENERATION_ONLY</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11RequestType28REQUEST_TYPE_GENERATION_ONLYE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryTypeE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aab"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType4kCPUE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType4kCPUE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType4kCPUE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aaba2ce310327f474afc9f6774faa2f57903"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCPU</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kCPUE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType11kCPU_PINNEDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType11kCPU_PINNEDE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType11kCPU_PINNEDE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aabaa12edb33c24c693b60e591681cfb1e66"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCPU_PINNED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType11kCPU_PINNEDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType15kCPU_PINNEDPOOLE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType15kCPU_PINNEDPOOLE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType15kCPU_PINNEDPOOLE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aabaf1e403dae17a973d2e1558c16e1cd1ef"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCPU_PINNEDPOOL</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType15kCPU_PINNEDPOOLE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType4kGPUE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType4kGPUE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType4kGPUE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aaba37ae524b76587efa776affdc5cdf2ac1"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGPU</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kGPUE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType4kUVME">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType4kUVME"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType4kUVME"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aaba7d7de0bd70e1276255098b25010bdeb6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUVM</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kUVME" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10MemoryType8kUNKNOWNE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10MemoryType8kUNKNOWNE"></span><span id="_CPPv2N12tensorrt_llm8executor10MemoryType8kUNKNOWNE"></span><span class="target" id="types_8h_1a8399248ba3babb51a2d0802a17e52aaba16bb40b9102367393ecf8213078d1c6e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUNKNOWN</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10MemoryType8kUNKNOWNE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9ModelTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9ModelTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor9ModelTypeE"></span><span class="target" id="types_8h_1ac442c56f38328a24c7fb20bdd7543c46"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModelType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9ModelType13kDECODER_ONLYE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9ModelType13kDECODER_ONLYE"></span><span id="_CPPv2N12tensorrt_llm8executor9ModelType13kDECODER_ONLYE"></span><span class="target" id="types_8h_1ac442c56f38328a24c7fb20bdd7543c46a577f62bbac6fac26ae5fcbfbd638405b"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDECODER_ONLY</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9ModelType13kDECODER_ONLYE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9ModelType13kENCODER_ONLYE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9ModelType13kENCODER_ONLYE"></span><span id="_CPPv2N12tensorrt_llm8executor9ModelType13kENCODER_ONLYE"></span><span class="target" id="types_8h_1ac442c56f38328a24c7fb20bdd7543c46ab9f0d192962e27b51920b525cfc0c8a6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kENCODER_ONLY</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9ModelType13kENCODER_ONLYE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor9ModelType16kENCODER_DECODERE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor9ModelType16kENCODER_DECODERE"></span><span id="_CPPv2N12tensorrt_llm8executor9ModelType16kENCODER_DECODERE"></span><span class="target" id="types_8h_1ac442c56f38328a24c7fb20bdd7543c46a9faae00b617c7bb72d509e32454d1758"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kENCODER_DECODER</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor9ModelType16kENCODER_DECODERE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12BatchingTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12BatchingTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor12BatchingTypeE"></span><span class="target" id="types_8h_1aca90eaee2e69172d309f77a581b824e3"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BatchingType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The batching type. </p>
|
||
<p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12BatchingType7kSTATICE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12BatchingType7kSTATICE"></span><span id="_CPPv2N12tensorrt_llm8executor12BatchingType7kSTATICE"></span><span class="target" id="types_8h_1aca90eaee2e69172d309f77a581b824e3a201894ba187a596f534f6406676f96bf"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kSTATIC</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12BatchingType7kSTATICE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>STATIC refers to the traditional batching scheme with a batch of requests running in lockstep until the full generation for all of them is complete. Requests in a batch are all padded up to the maximum input and output sequence length of any member of the batch. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12BatchingType9kINFLIGHTE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12BatchingType9kINFLIGHTE"></span><span id="_CPPv2N12tensorrt_llm8executor12BatchingType9kINFLIGHTE"></span><span class="target" id="types_8h_1aca90eaee2e69172d309f77a581b824e3af9f499a6c3b205667d7f5ddba6bf4c02"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kINFLIGHT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12BatchingType9kINFLIGHTE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>INFLIGHT refers to a scheme where newly arrived requests are dynamically incorporated into the batch under execution, and requests are returned as soon as the end condition is met without any padding. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23CapacitySchedulerPolicyE"></span><span id="_CPPv2N12tensorrt_llm8executor23CapacitySchedulerPolicyE"></span><span class="target" id="types_8h_1aae89f79e4f8c4a402fa826e3acf68428"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The policy used to select the subset of available requests in each iteration of the executor generation loop. </p>
|
||
<p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy16kMAX_UTILIZATIONE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23CapacitySchedulerPolicy16kMAX_UTILIZATIONE"></span><span id="_CPPv2N12tensorrt_llm8executor23CapacitySchedulerPolicy16kMAX_UTILIZATIONE"></span><span class="target" id="types_8h_1aae89f79e4f8c4a402fa826e3acf68428ae7df1f3a699ee2bfbe4328f94e825b7e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMAX_UTILIZATION</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy16kMAX_UTILIZATIONE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>MAX_UTILIZATION packs as many requests as the underlying TRT engine can support in any iteration of the InflightBatching generation loop. While this is expected to maximize GPU throughput, it might require that some requests be paused and restarted depending on peak KV cache memory availability. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE"></span><span id="_CPPv2N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE"></span><span class="target" id="types_8h_1aae89f79e4f8c4a402fa826e3acf68428a880f95ca896ec21f5e0a981aac04d10b"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGUARANTEED_NO_EVICT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>GUARANTEED_NO_EVICT uses KV cache more conservatively guaranteeing that a request, once started, will run to completion without eviction. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy13kSTATIC_BATCHE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23CapacitySchedulerPolicy13kSTATIC_BATCHE"></span><span id="_CPPv2N12tensorrt_llm8executor23CapacitySchedulerPolicy13kSTATIC_BATCHE"></span><span class="target" id="types_8h_1aae89f79e4f8c4a402fa826e3acf68428ac52787543a2afbe58114adaf73d0d1b6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kSTATIC_BATCH</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy13kSTATIC_BATCHE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>kSTATIC_BATCH does not schedule new requests until all requests in current batch are completed. Similar to kGUARANTEED_NO_EVICT, requests will run to completion without eviction. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21ContextChunkingPolicyE"></span><span id="_CPPv2N12tensorrt_llm8executor21ContextChunkingPolicyE"></span><span class="target" id="types_8h_1a6d4f91b57e2ce01f804855083e22a4bf"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ContextChunkingPolicy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy24kFIRST_COME_FIRST_SERVEDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21ContextChunkingPolicy24kFIRST_COME_FIRST_SERVEDE"></span><span id="_CPPv2N12tensorrt_llm8executor21ContextChunkingPolicy24kFIRST_COME_FIRST_SERVEDE"></span><span class="target" id="types_8h_1a6d4f91b57e2ce01f804855083e22a4bfac5d100effe3b0eee3f5b4d48d4b5a5a9"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kFIRST_COME_FIRST_SERVED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy24kFIRST_COME_FIRST_SERVEDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Sequential chunking, complete the unfinished context phase first. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy15kEQUAL_PROGRESSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21ContextChunkingPolicy15kEQUAL_PROGRESSE"></span><span id="_CPPv2N12tensorrt_llm8executor21ContextChunkingPolicy15kEQUAL_PROGRESSE"></span><span class="target" id="types_8h_1a6d4f91b57e2ce01f804855083e22a4bfac263978419a580b4ef16a199b050f89c"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEQUAL_PROGRESS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy15kEQUAL_PROGRESSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Iterate through each context request in sequence and attempt to increase its chunk count until the constraint is exceeded. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17CommunicationTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17CommunicationTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor17CommunicationTypeE"></span><span class="target" id="types_8h_1a93b2af939dbc2cc1690fb10232675ba0"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommunicationType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17CommunicationType4kMPIE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17CommunicationType4kMPIE"></span><span id="_CPPv2N12tensorrt_llm8executor17CommunicationType4kMPIE"></span><span class="target" id="types_8h_1a93b2af939dbc2cc1690fb10232675ba0a4352f81c41aaa6b77e87c062d4347e2b"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMPI</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17CommunicationType4kMPIE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17CommunicationModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17CommunicationModeE"></span><span id="_CPPv2N12tensorrt_llm8executor17CommunicationModeE"></span><span class="target" id="types_8h_1ace2966b11ce94befb6438957e8aef1a4"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommunicationMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17CommunicationMode7kLEADERE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17CommunicationMode7kLEADERE"></span><span id="_CPPv2N12tensorrt_llm8executor17CommunicationMode7kLEADERE"></span><span class="target" id="types_8h_1ace2966b11ce94befb6438957e8aef1a4a211ca40446bff2e65500e3e544664df7"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLEADER</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17CommunicationMode7kLEADERE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17CommunicationMode13kORCHESTRATORE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17CommunicationMode13kORCHESTRATORE"></span><span id="_CPPv2N12tensorrt_llm8executor17CommunicationMode13kORCHESTRATORE"></span><span class="target" id="types_8h_1ace2966b11ce94befb6438957e8aef1a4a88924e2c20e89e79d5782505f7228158"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kORCHESTRATOR</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17CommunicationMode13kORCHESTRATORE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStageE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStageE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097ca"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestStage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enum class that represents the state of a request. </p>
|
||
<p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStage7kQUEUEDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStage7kQUEUEDE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStage7kQUEUEDE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097caaa2247736a3991a2eb2fe91dd90b2f6af"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kQUEUED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStage7kQUEUEDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1Request"><span class="std std-ref">Request</span></a> that have been received but not yet included in the active requests (due to constraints such as maximum batch size for example). </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStage20kENCODER_IN_PROGRESSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStage20kENCODER_IN_PROGRESSE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStage20kENCODER_IN_PROGRESSE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097caaf777328746e427f4925d2423d7722757"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kENCODER_IN_PROGRESS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kENCODER_IN_PROGRESSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Active request in encoder phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStage20kCONTEXT_IN_PROGRESSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStage20kCONTEXT_IN_PROGRESSE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStage20kCONTEXT_IN_PROGRESSE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097caa0d34fe21e6c4aeae3dc8b16b2626ad0a"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCONTEXT_IN_PROGRESS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kCONTEXT_IN_PROGRESSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Active request in context phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStage23kGENERATION_IN_PROGRESSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStage23kGENERATION_IN_PROGRESSE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStage23kGENERATION_IN_PROGRESSE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097caa00590ca88f1c0cfb06a0498c299476d4"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGENERATION_IN_PROGRESS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStage23kGENERATION_IN_PROGRESSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Active request in generation phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStage20kGENERATION_COMPLETEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStage20kGENERATION_COMPLETEE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStage20kGENERATION_COMPLETEE"></span><span class="target" id="types_8h_1a557b39c71d92e2c17116c4972e5097caab437475b0e4a8e154f8af3ba70c51dc3"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGENERATION_COMPLETE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kGENERATION_COMPLETEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Active request for which generation has completed. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReasonE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReasonE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReasonE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FinishReason</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReasonE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The reason why the model stopped generating tokens for a request. </p>
|
||
<p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason13kNOT_FINISHEDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason13kNOT_FINISHEDE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason13kNOT_FINISHEDE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6a2e35cacb9bb2d1aeeeccade227905c0e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kNOT_FINISHED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason13kNOT_FINISHEDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request is not finished. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason7kEND_IDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason7kEND_IDE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason7kEND_IDE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6a8dd28f2cf09133dd1f5faee718454414"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEND_ID</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason7kEND_IDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request finished because the end id was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason11kSTOP_WORDSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason11kSTOP_WORDSE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason11kSTOP_WORDSE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6a2ab216f49c325a41266959adae89d820"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kSTOP_WORDS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason11kSTOP_WORDSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request finished because a stop word was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason7kLENGTHE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason7kLENGTHE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason7kLENGTHE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6ad9ab945eba83fecb2c324057275c0876"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLENGTH</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason7kLENGTHE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request finished because the maximum number of tokens was reached. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason10kTIMED_OUTE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason10kTIMED_OUTE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason10kTIMED_OUTE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6ad470dffc98a09c6158718c8827de10b4"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kTIMED_OUT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason10kTIMED_OUTE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request finished because it got timed out (via the mAllotedTime parameter) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12FinishReason10kCANCELLEDE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12FinishReason10kCANCELLEDE"></span><span id="_CPPv2N12tensorrt_llm8executor12FinishReason10kCANCELLEDE"></span><span class="target" id="types_8h_1a224950d1b5760e0c752649295b84ada6a0db9706b012c1c09997a42ef9fb76791"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCANCELLED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12FinishReason10kCANCELLEDE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request was cancelled by calling cancelRequest. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KvCacheTransferModeE"></span><span id="_CPPv2N12tensorrt_llm8executor19KvCacheTransferModeE"></span><span class="target" id="types_8h_1a345a3e90232624cc0c5adf7896ca273e"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheTransferMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enum describing the transfer mode for KV cache. </p>
|
||
<p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode4DRAME">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KvCacheTransferMode4DRAME"></span><span id="_CPPv2N12tensorrt_llm8executor19KvCacheTransferMode4DRAME"></span><span class="target" id="types_8h_1a345a3e90232624cc0c5adf7896ca273eaebae17841ce69e653df838d8c20ace8d"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DRAM</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode4DRAME" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Copy to/from CPU memory (original approach). </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode3GDSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KvCacheTransferMode3GDSE"></span><span id="_CPPv2N12tensorrt_llm8executor19KvCacheTransferMode3GDSE"></span><span class="target" id="types_8h_1a345a3e90232624cc0c5adf7896ca273ea495ebb08d96fd1ef02a05992b75a58e0"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GDS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode3GDSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Attempt GPUDirect Storage (cuFile). </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode20POSIX_DEBUG_FALLBACKE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KvCacheTransferMode20POSIX_DEBUG_FALLBACKE"></span><span id="_CPPv2N12tensorrt_llm8executor19KvCacheTransferMode20POSIX_DEBUG_FALLBACKE"></span><span class="target" id="types_8h_1a345a3e90232624cc0c5adf7896ca273ea61d064a0dc44ecccf71241b373282ebe"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">POSIX_DEBUG_FALLBACK</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode20POSIX_DEBUG_FALLBACKE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Force a POSIX read/write for debugging. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE23CapacitySchedulerPolicy">
|
||
<span id="_CPPv3N12tensorrt_llm8executorlsERNSt7ostreamE23CapacitySchedulerPolicy"></span><span id="_CPPv2N12tensorrt_llm8executorlsERNSt7ostreamE23CapacitySchedulerPolicy"></span><span id="tensorrt_llm::executor::lshift-operator__osR.CapacitySchedulerPolicy"></span><span class="target" id="types_8h_1a7e30017998937539ebe04d46a822ef5b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre"><<</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="tensorrt_llm::executor::CapacitySchedulerPolicy"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">policy</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE23CapacitySchedulerPolicy" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE21ContextChunkingPolicy">
|
||
<span id="_CPPv3N12tensorrt_llm8executorlsERNSt7ostreamE21ContextChunkingPolicy"></span><span id="_CPPv2N12tensorrt_llm8executorlsERNSt7ostreamE21ContextChunkingPolicy"></span><span id="tensorrt_llm::executor::lshift-operator__osR.ContextChunkingPolicy"></span><span class="target" id="types_8h_1a314256f0336f928899256adc3cb97145"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre"><<</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE" title="tensorrt_llm::executor::ContextChunkingPolicy"><span class="n"><span class="pre">ContextChunkingPolicy</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">policy</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE21ContextChunkingPolicy" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIterationE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24DebugTensorsPerIterationE"></span><span id="_CPPv2N12tensorrt_llm8executor24DebugTensorsPerIterationE"></span><span id="tensorrt_llm::executor::DebugTensorsPerIteration"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DebugTensorsPerIteration"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DebugTensorsPerIteration</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIterationE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the debug tensors in an iteration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration4iterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24DebugTensorsPerIteration4iterE"></span><span id="_CPPv2N12tensorrt_llm8executor24DebugTensorsPerIteration4iterE"></span><span id="tensorrt_llm::executor::DebugTensorsPerIteration::iter__IterationType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DebugTensorsPerIteration_1a27303becd7f64dac5c3b36c0830151ee"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration4iterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The iteration id for these tensors. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration12debugTensorsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24DebugTensorsPerIteration12debugTensorsE"></span><span id="_CPPv2N12tensorrt_llm8executor24DebugTensorsPerIteration12debugTensorsE"></span><span id="tensorrt_llm::executor::DebugTensorsPerIteration::debugTensors__std::map:ss.Tensor:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DebugTensorsPerIteration_1acba4a2160ca6f95df41e635d95780a7f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">debugTensors</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration12debugTensorsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The debug tensors for this iteration. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingModeE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingModeE"></span><span id="tensorrt_llm::executor::DecodingMode"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>mode of the decoder </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a3448ce5ea22fc30852625ab0fef44d7c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UnderlyingType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">uint32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode14useTemperatureEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode14useTemperatureEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode14useTemperatureEb"></span><span id="tensorrt_llm::executor::DecodingMode::useTemperature__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a53684be976895eebc1664f0d6c70ae6a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useTemperature</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useTemp</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14useTemperatureEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode22useOccurrencePenaltiesEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode22useOccurrencePenaltiesEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode22useOccurrencePenaltiesEb"></span><span id="tensorrt_llm::executor::DecodingMode::useOccurrencePenalties__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1adbefa253ae6fa8818681b7cb8a4ec0f9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useOccurrencePenalties</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">usePenalty</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode22useOccurrencePenaltiesEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode18usePresencePenaltyEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode18usePresencePenaltyEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode18usePresencePenaltyEb"></span><span id="tensorrt_llm::executor::DecodingMode::usePresencePenalty__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ac7788f151b7ad6f02d3accfcbc998d8b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePresencePenalty</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">usePenalty</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode18usePresencePenaltyEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode20useRepetitionPenaltyEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode20useRepetitionPenaltyEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode20useRepetitionPenaltyEb"></span><span id="tensorrt_llm::executor::DecodingMode::useRepetitionPenalty__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1af96b575f7955dc093b069fd43b55a634"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useRepetitionPenalty</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">usePenalty</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20useRepetitionPenaltyEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode19useFrequencyPenaltyEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode19useFrequencyPenaltyEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode19useFrequencyPenaltyEb"></span><span id="tensorrt_llm::executor::DecodingMode::useFrequencyPenalty__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1af27a60691183d1fababc834943b0a0da"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useFrequencyPenalty</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">usePenalty</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19useFrequencyPenaltyEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode12useMinLengthEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode12useMinLengthEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode12useMinLengthEb"></span><span id="tensorrt_llm::executor::DecodingMode::useMinLength__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a87b550d95c6317042ddafd47919e3dd6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMinLength</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useMinLen</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useMinLengthEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode12useBanTokensEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode12useBanTokensEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode12useBanTokensEb"></span><span id="tensorrt_llm::executor::DecodingMode::useBanTokens__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1ab86e40f9cac2d7f9bc7f0e289a50c6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useBanTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">banTokens</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useBanTokensEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode11useBanWordsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode11useBanWordsEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode11useBanWordsEb"></span><span id="tensorrt_llm::executor::DecodingMode::useBanWords__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a3df6e7d8e5897accecaa4020754f8907"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useBanWords</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">banWords</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode11useBanWordsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode20useNoRepeatNgramSizeEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode20useNoRepeatNgramSizeEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode20useNoRepeatNgramSizeEb"></span><span id="tensorrt_llm::executor::DecodingMode::useNoRepeatNgramSize__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1afd8cb08a97e7ef3d7d22a2ef9a5da52d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useNoRepeatNgramSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">noRepeatNgramSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20useNoRepeatNgramSizeEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode12useStopWordsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode12useStopWordsEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode12useStopWordsEb"></span><span id="tensorrt_llm::executor::DecodingMode::useStopWords__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1abb9e70c7d442d0ff4cb7fd4520f5d2ed"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useStopWords</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stopWords</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useStopWordsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode16useMaxLengthStopEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode16useMaxLengthStopEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode16useMaxLengthStopEb"></span><span id="tensorrt_llm::executor::DecodingMode::useMaxLengthStop__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a5b185534052f4c92bfbc0656347dc815"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMaxLengthStop</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxLengthStop</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode16useMaxLengthStopEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode18useExplicitEosStopEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode18useExplicitEosStopEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode18useExplicitEosStopEb"></span><span id="tensorrt_llm::executor::DecodingMode::useExplicitEosStop__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a7619fa8157c8ae7a7f14080d9658b07f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useExplicitEosStop</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">explicitEosStop</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode18useExplicitEosStopEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode7useMinPEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode7useMinPEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode7useMinPEb"></span><span id="tensorrt_llm::executor::DecodingMode::useMinP__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a416e051052d8d13657472d5805afdf1f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMinP</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useMinP</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode7useMinPEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode26useVariableBeamWidthSearchEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode26useVariableBeamWidthSearchEb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode26useVariableBeamWidthSearchEb"></span><span id="tensorrt_llm::executor::DecodingMode::useVariableBeamWidthSearch__bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a8c9b9111297ddb775365cd8bb3a17ab5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useVariableBeamWidthSearch</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useVariableBeamWidthSearch</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode26useVariableBeamWidthSearchEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode6isAutoEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode6isAutoEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode6isAutoEv"></span><span id="tensorrt_llm::executor::DecodingMode::isAutoCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a13a8c66dca42a90bbf87f9b1dd2ff975"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isAuto</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isAutoEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopKEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode6isTopKEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode6isTopKEv"></span><span id="tensorrt_llm::executor::DecodingMode::isTopKCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1acef6cd75fa3225f59b93bf5b37f5091e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTopK</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopKEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode6isTopPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode6isTopPEv"></span><span id="tensorrt_llm::executor::DecodingMode::isTopPCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a6d89a1dc788260a68bb31cf6d914afe7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode12isTopKorTopPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode12isTopKorTopPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode12isTopKorTopPEv"></span><span id="tensorrt_llm::executor::DecodingMode::isTopKorTopPCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aa63cc1be02bcc70d2984e362a3eed173"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTopKorTopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isTopKorTopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode13isTopKandTopPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode13isTopKandTopPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode13isTopKandTopPEv"></span><span id="tensorrt_llm::executor::DecodingMode::isTopKandTopPCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a311be6392cb57c618401c85435471a9f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTopKandTopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode13isTopKandTopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode12isBeamSearchEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode12isBeamSearchEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode12isBeamSearchEv"></span><span id="tensorrt_llm::executor::DecodingMode::isBeamSearchCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aca93dead027944e2dce0aee783af1805"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isBeamSearch</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isBeamSearchEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode8isMedusaEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode8isMedusaEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode8isMedusaEv"></span><span id="tensorrt_llm::executor::DecodingMode::isMedusaCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a918a24ee9837d1d4445f993f58e87473"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isMedusa</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode8isMedusaEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode11isLookaheadEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode11isLookaheadEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode11isLookaheadEv"></span><span id="tensorrt_llm::executor::DecodingMode::isLookaheadCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1c23e7c880b6b756654aada19882be64"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLookahead</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode11isLookaheadEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExplicitDraftTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode21isExplicitDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode21isExplicitDraftTokensEv"></span><span id="tensorrt_llm::executor::DecodingMode::isExplicitDraftTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a5cf22aa39532d314b197f32243506ca7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isExplicitDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExplicitDraftTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExternalDraftTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode21isExternalDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode21isExternalDraftTokensEv"></span><span id="tensorrt_llm::executor::DecodingMode::isExternalDraftTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a9369abde5d01be56b863019dd4566372"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isExternalDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExternalDraftTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode7isEagleEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode7isEagleEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode7isEagleEv"></span><span id="tensorrt_llm::executor::DecodingMode::isEagleCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aacea10c88d6ed17b9e0afb30ba25c93c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isEagle</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode7isEagleEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode16isUseTemperatureEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode16isUseTemperatureEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode16isUseTemperatureEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseTemperatureCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1f9428f57c22e7f0ef830093c871443c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseTemperature</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode16isUseTemperatureEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUsePresencePenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode20isUsePresencePenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode20isUsePresencePenaltyEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUsePresencePenaltyCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a117bd359894342c3d47f5aced212c8e0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUsePresencePenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUsePresencePenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode21isUseFrequencyPenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode21isUseFrequencyPenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode21isUseFrequencyPenaltyEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseFrequencyPenaltyCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1cffde64810926fefcbd62248dc57f13"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseFrequencyPenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isUseFrequencyPenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseRepetitionPenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode22isUseRepetitionPenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode22isUseRepetitionPenaltyEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseRepetitionPenaltyCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a0f1b4906c1e8da7da08655e9d95460bb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseRepetitionPenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseRepetitionPenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseMinLengthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode14isUseMinLengthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode14isUseMinLengthEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseMinLengthCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ab570344af6e79189256fdd2990de50f1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseMinLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseMinLengthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseOccurrencePenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode22isUseOccurrencePenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode22isUseOccurrencePenaltyEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseOccurrencePenaltyCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a3366394589b741680cf971361e3b1a03"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseOccurrencePenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseOccurrencePenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode12isUsePenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode12isUsePenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode12isUsePenaltyEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUsePenaltyCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a46f7c33f03d6bc54569f0b5ff74d64fd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUsePenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isUsePenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode13isUseBanWordsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode13isUseBanWordsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode13isUseBanWordsEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseBanWordsCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1abdca9b5b5b708624d9a07a5530492c3c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseBanWords</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode13isUseBanWordsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseNoRepeatNgramSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode22isUseNoRepeatNgramSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode22isUseNoRepeatNgramSizeEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseNoRepeatNgramSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ab521096f583e2d773acad9d86e1d1925"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseNoRepeatNgramSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseNoRepeatNgramSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseBanTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode14isUseBanTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode14isUseBanTokensEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseBanTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1acc4cf4d789890aa53d21de141d9e898d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseBanTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseBanTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseStopWordsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode14isUseStopWordsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode14isUseStopWordsEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseStopWordsCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a943fa69829d69a7158c836daaa232e77"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseStopWords</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseStopWordsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode18isUseMaxLengthStopEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode18isUseMaxLengthStopEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode18isUseMaxLengthStopEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseMaxLengthStopCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ada5840f7445fc38d13912b6c7b9806bd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseMaxLengthStop</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode18isUseMaxLengthStopEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUseExplicitEosStopEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode20isUseExplicitEosStopEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode20isUseExplicitEosStopEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseExplicitEosStopCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aad01d4a47d716f65ed97dcace6d00d75"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseExplicitEosStop</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUseExplicitEosStopEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode17isUseStopCriteriaEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode17isUseStopCriteriaEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode17isUseStopCriteriaEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseStopCriteriaCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a75e451f8c7cff66006074a2e5aa9c493"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseStopCriteria</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode17isUseStopCriteriaEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode9isUseMinPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode9isUseMinPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode9isUseMinPEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseMinPCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a5fc52bf3d51c226f15626038649f96de"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseMinP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9isUseMinPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode28isUseVariableBeamWidthSearchEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode28isUseVariableBeamWidthSearchEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode28isUseVariableBeamWidthSearchEv"></span><span id="tensorrt_llm::executor::DecodingMode::isUseVariableBeamWidthSearchCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1af819b6b362c6a7b0caaa1a71756651aa"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUseVariableBeamWidthSearch</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode28isUseVariableBeamWidthSearchEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingModeeqERK12DecodingMode">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingModeeqERK12DecodingMode"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingModeeqERK12DecodingMode"></span><span id="tensorrt_llm::executor::DecodingMode::eq-operator__DecodingModeCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aaa854d9caaad142db1f8f0cbfcba8f8f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingModeeqERK12DecodingMode" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode12DecodingModeE14UnderlyingType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode12DecodingModeE14UnderlyingType"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode12DecodingModeE14UnderlyingType"></span><span id="tensorrt_llm::executor::DecodingMode::DecodingMode__UnderlyingTypeCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a844ac72db3cf0d3961dc9a90f86e101b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12DecodingModeE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode8getStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode8getStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode8getStateEv"></span><span id="tensorrt_llm::executor::DecodingMode::getStateCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a27d6ca8f5f71b63b26dc7eb2b3d03cfc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode8getStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode7getNameEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode7getNameEv"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode7getNameEv"></span><span id="tensorrt_llm::executor::DecodingMode::getNameCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a7344f35edad79878bca311680a876d5b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode7getNameEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode4AutoEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode4AutoEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode4AutoEv"></span><span id="tensorrt_llm::executor::DecodingMode::AutoCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ae19b52345b0ceaa53893604fffd8f0e9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Auto</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4AutoEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>No mode specified. Config will be determined from the beam width of the first request at runtime TopKTopP if beamWidth == 1, BeamSearch otherwise. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode4TopKEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode4TopKEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode4TopKEv"></span><span id="tensorrt_llm::executor::DecodingMode::TopKCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aa373bed34ebe0279642f5c040e531d2d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TopK</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4TopKEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode4TopPEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode4TopPEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode4TopPEv"></span><span id="tensorrt_llm::executor::DecodingMode::TopPCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a06af714392ace9ed4a8e12a929f90436"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4TopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode8TopKTopPEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode8TopKTopPEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode8TopKTopPEv"></span><span id="tensorrt_llm::executor::DecodingMode::TopKTopPCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a6357866b6b060de43e847377ced8cfab"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TopKTopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8TopKTopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode10BeamSearchEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode10BeamSearchEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode10BeamSearchEv"></span><span id="tensorrt_llm::executor::DecodingMode::BeamSearchCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a37f194965847988e8a432ada9d22de66"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BeamSearch</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode10BeamSearchEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode6MedusaEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode6MedusaEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode6MedusaEv"></span><span id="tensorrt_llm::executor::DecodingMode::MedusaCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a534c86f3ba680562b04f0a859027e019"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Medusa</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6MedusaEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode9LookaheadEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode9LookaheadEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode9LookaheadEv"></span><span id="tensorrt_llm::executor::DecodingMode::LookaheadCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1abfa9bfe2f07541d845c8c4e9967eeac7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Lookahead</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9LookaheadEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode19ExplicitDraftTokensEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode19ExplicitDraftTokensEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode19ExplicitDraftTokensEv"></span><span id="tensorrt_llm::executor::DecodingMode::ExplicitDraftTokensCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a19ec77f227684778f24dceb21413a8ac"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExplicitDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19ExplicitDraftTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode19ExternalDraftTokensEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode19ExternalDraftTokensEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode19ExternalDraftTokensEv"></span><span id="tensorrt_llm::executor::DecodingMode::ExternalDraftTokensCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aa640976fea3f031ebac9d768f522df91"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExternalDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19ExternalDraftTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode5EagleEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode5EagleEv"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode5EagleEv"></span><span id="tensorrt_llm::executor::DecodingMode::EagleCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a3a3b86b6930f3a6f29070f25bcfa0510"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Eagle</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5EagleEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode9anyBitSetE14UnderlyingType">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode9anyBitSetE14UnderlyingType"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode9anyBitSetE14UnderlyingType"></span><span id="tensorrt_llm::executor::DecodingMode::anyBitSet__UnderlyingTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a32ede164c41b093aae7c85b3219e5f17"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">anyBitSet</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bits</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9anyBitSetE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor12DecodingMode9allBitSetE14UnderlyingType">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor12DecodingMode9allBitSetE14UnderlyingType"></span><span id="_CPPv2NK12tensorrt_llm8executor12DecodingMode9allBitSetE14UnderlyingType"></span><span id="tensorrt_llm::executor::DecodingMode::allBitSet__UnderlyingTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a26b5b3ca9c8a51fe1c80634d88467fdf"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allBitSet</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bits</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9allBitSetE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode8setBitToE14UnderlyingTypeb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode8setBitToE14UnderlyingTypeb"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode8setBitToE14UnderlyingTypeb"></span><span id="tensorrt_llm::executor::DecodingMode::setBitTo__UnderlyingType.bCE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a6d9fd972c8e6732431571cf413d9eb96"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBitTo</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">x</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8setBitToE14UnderlyingTypeb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode6mStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode6mStateE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode6mStateE"></span><span id="tensorrt_llm::executor::DecodingMode::mState__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a7d858720e8c48b761bfb6fae613cd69f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mState</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6mStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-attributes">Private Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode9kNumFlagsE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode9kNumFlagsE"></span><span id="tensorrt_llm::executor::DecodingMode::kNumFlags__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a4ccbf3895859431cc61a3390a2ea77fb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kNumFlags</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">12</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseRepetitionPenalties__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a56fd74b99c930baabf3cec5ee5064781"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseRepetitionPenalties</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseFrequencyPenalties__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a87b52408c776e9e46b80cfc53973d131"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseFrequencyPenalties</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE"></span><span id="tensorrt_llm::executor::DecodingMode::kUsePresencePenalties__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a62bbed2ed4e23bd133a50b8478f0a340"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUsePresencePenalties</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseTemperature__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a68b45f890a7ce1a6c690b6450bd90775"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseTemperature</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseMinLength__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a505e61c877fb6e1a0f55bd97e190c387"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseMinLength</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseBanWords__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a99e799e22a27fae5470f85b56d3b4350"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseBanWords</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseStopWords__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ae0b84d84d79681471b066d0ef57ee186"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseStopWords</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseMaxLengthStop__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aca8d72e75f44aa7884f8201bef2c9b13"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseMaxLengthStop</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode19kUseExplicitEosStopE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode19kUseExplicitEosStopE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode19kUseExplicitEosStopE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseExplicitEosStop__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a50b13f8199ae42fbd7889f04acb4bd50"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseExplicitEosStop</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19kUseExplicitEosStopE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseNoRepeatNgramSize__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ae715ea3674c97b0d768c779cb763866d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseNoRepeatNgramSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode8kUseMinPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode8kUseMinPE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode8kUseMinPE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseMinP__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a0173031b393c6383e322e8923b298810"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseMinP</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">10</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8kUseMinPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode27kUseVariableBeamWidthSearchE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode27kUseVariableBeamWidthSearchE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode27kUseVariableBeamWidthSearchE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseVariableBeamWidthSearch__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a4d3b18f739cb61958cc1a460e23087b5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseVariableBeamWidthSearch</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="m"><span class="pre">11</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode27kUseVariableBeamWidthSearchE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode24kUseStandardStopCriteriaE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode24kUseStandardStopCriteriaE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode24kUseStandardStopCriteriaE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseStandardStopCriteria__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aec9506a9ecb0b1f1f5ccab5acc6abfcd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseStandardStopCriteria</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE" title="tensorrt_llm::executor::DecodingMode::kUseStopWords"><span class="n"><span class="pre">kUseStopWords</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE" title="tensorrt_llm::executor::DecodingMode::kUseMaxLengthStop"><span class="n"><span class="pre">kUseMaxLengthStop</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode24kUseStandardStopCriteriaE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseOccurrencePenalties__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ab96b961e00ac3efd8cea889aac42e7ed"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseOccurrencePenalties</span></span></span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE" title="tensorrt_llm::executor::DecodingMode::kUseRepetitionPenalties"><span class="n"><span class="pre">kUseRepetitionPenalties</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE" title="tensorrt_llm::executor::DecodingMode::kUseFrequencyPenalties"><span class="n"><span class="pre">kUseFrequencyPenalties</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE" title="tensorrt_llm::executor::DecodingMode::kUsePresencePenalties"><span class="n"><span class="pre">kUsePresencePenalties</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode13kUsePenaltiesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode13kUsePenaltiesE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode13kUsePenaltiesE"></span><span id="tensorrt_llm::executor::DecodingMode::kUsePenalties__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1f64b7fd17705346396cc54b8e0898ea"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUsePenalties</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE" title="tensorrt_llm::executor::DecodingMode::kUseOccurrencePenalties"><span class="n"><span class="pre">kUseOccurrencePenalties</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE" title="tensorrt_llm::executor::DecodingMode::kUseTemperature"><span class="n"><span class="pre">kUseTemperature</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE" title="tensorrt_llm::executor::DecodingMode::kUseMinLength"><span class="n"><span class="pre">kUseMinLength</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUsePenaltiesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseBanTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode13kUseBanTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode13kUseBanTokensE"></span><span id="tensorrt_llm::executor::DecodingMode::kUseBanTokens__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aec58c8d8748972a10f900b0cd2899afa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUseBanTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE" title="tensorrt_llm::executor::DecodingMode::kUseNoRepeatNgramSize"><span class="n"><span class="pre">kUseNoRepeatNgramSize</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE" title="tensorrt_llm::executor::DecodingMode::kUseBanWords"><span class="n"><span class="pre">kUseBanWords</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseBanTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode5kAutoE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode5kAutoE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode5kAutoE"></span><span id="tensorrt_llm::executor::DecodingMode::kAuto__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1ab447608fe495937145568cf45fedaf8e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kAuto</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kAutoE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopKE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode5kTopKE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode5kTopKE"></span><span id="tensorrt_llm::executor::DecodingMode::kTopK__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a1b4479a1017694d10148f310055f3d19"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kTopK</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopKE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode5kTopPE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode5kTopPE"></span><span id="tensorrt_llm::executor::DecodingMode::kTopP__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a321db8b7aa9a93a6bb2af6580d060b6e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kTopP</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode11kBeamSearchE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode11kBeamSearchE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode11kBeamSearchE"></span><span id="tensorrt_llm::executor::DecodingMode::kBeamSearch__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a78a537977a0d6c09d510c3442d389647"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBeamSearch</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode11kBeamSearchE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode7kMedusaE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode7kMedusaE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode7kMedusaE"></span><span id="tensorrt_llm::executor::DecodingMode::kMedusa__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a7e480511b5f4e67bfae949ad42d3a7d1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMedusa</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode7kMedusaE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode10kLookaheadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode10kLookaheadE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode10kLookaheadE"></span><span id="tensorrt_llm::executor::DecodingMode::kLookahead__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1aaae72084fe5bb0a0cf7a93acee8ac0ba"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLookahead</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode10kLookaheadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode20kExplicitDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode20kExplicitDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode20kExplicitDraftTokensE"></span><span id="tensorrt_llm::executor::DecodingMode::kExplicitDraftTokens__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a23bc169569946f8eb079164d0d39ab0b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kExplicitDraftTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20kExplicitDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode20kExternalDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode20kExternalDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode20kExternalDraftTokensE"></span><span id="tensorrt_llm::executor::DecodingMode::kExternalDraftTokens__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a280281eda5b35b040985638b2228ac35"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kExternalDraftTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20kExternalDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode6kEagleE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode6kEagleE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode6kEagleE"></span><span id="tensorrt_llm::executor::DecodingMode::kEagle__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a9c932c52575cc4cc3949de88f9a66d07"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEagle</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1u</span></span><span class="w"> </span><span class="o"><span class="pre"><<</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE" title="tensorrt_llm::executor::DecodingMode::kNumFlags"><span class="n"><span class="pre">kNumFlags</span></span></a><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6kEagleE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12DecodingMode9kTopKTopPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12DecodingMode9kTopKTopPE"></span><span id="_CPPv2N12tensorrt_llm8executor12DecodingMode9kTopKTopPE"></span><span id="tensorrt_llm::executor::DecodingMode::kTopKTopP__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingMode_1a8f4f53c2825bdbee88983507f0c2f413"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE" title="tensorrt_llm::executor::DecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kTopKTopP</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopKE" title="tensorrt_llm::executor::DecodingMode::kTopK"><span class="n"><span class="pre">kTopK</span></span></a><span class="w"> </span><span class="o"><span class="pre">|</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopPE" title="tensorrt_llm::executor::DecodingMode::kTopP"><span class="n"><span class="pre">kTopP</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kTopKTopPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22DisServingRequestStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor22DisServingRequestStatsE"></span><span id="tensorrt_llm::executor::DisServingRequestStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DisServingRequestStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DisServingRequestStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the request stats in the case of disaggregated serving. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22DisServingRequestStats17kvCacheTransferMSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22DisServingRequestStats17kvCacheTransferMSE"></span><span id="_CPPv2N12tensorrt_llm8executor22DisServingRequestStats17kvCacheTransferMSE"></span><span id="tensorrt_llm::executor::DisServingRequestStats::kvCacheTransferMS__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DisServingRequestStats_1a5f3f1be5b29fb05aea1f8bd97b9ce0e6"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheTransferMS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStats17kvCacheTransferMSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The total time spent on transferring KV cache from context phase to generation phase (ms) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22DisServingRequestStats11kvCacheSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22DisServingRequestStats11kvCacheSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor22DisServingRequestStats11kvCacheSizeE"></span><span id="tensorrt_llm::executor::DisServingRequestStats::kvCacheSize__s"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1DisServingRequestStats_1a1ea491fc36d566aedd08295402b4fae4"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStats11kvCacheSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The total size of KV cache transferred from context phase to generation phase (bytes) </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStatsE"></span><span id="tensorrt_llm::executor::InflightBatchingStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">InflightBatchingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of inflight batching models for a single iteration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats20numScheduledRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats20numScheduledRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats20numScheduledRequestsE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::numScheduledRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1adf6843f2538709caad5542216b13693e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numScheduledRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats20numScheduledRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of scheduled requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats18numContextRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats18numContextRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats18numContextRequestsE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::numContextRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1afc34637bdddb02cb101f38a9c8a4a50a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numContextRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats18numContextRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of requests in context stage. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats14numGenRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats14numGenRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats14numGenRequestsE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::numGenRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1a226f8ec68000216bfeedc6040e08da7d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numGenRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats14numGenRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of requests in generation stage. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats17numPausedRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats17numPausedRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats17numPausedRequestsE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::numPausedRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1a942c50f8eaba7dd00ded9f10e257f286"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numPausedRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats17numPausedRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of paused requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12numCtxTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats12numCtxTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats12numCtxTokensE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::numCtxTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1a052d62534092ccc9824332a1d55da3a8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numCtxTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12numCtxTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of context tokens in the iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12microBatchIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats12microBatchIdE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats12microBatchIdE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::microBatchId__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1a945be330caa27de5a173aec6d71a6b03"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">microBatchId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12microBatchIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Index of mirco batch. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21InflightBatchingStats26avgNumDecodedTokensPerIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21InflightBatchingStats26avgNumDecodedTokensPerIterE"></span><span id="_CPPv2N12tensorrt_llm8executor21InflightBatchingStats26avgNumDecodedTokensPerIterE"></span><span id="tensorrt_llm::executor::InflightBatchingStats::avgNumDecodedTokensPerIter__float"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1InflightBatchingStats_1acca77d655808fe504c6f99c8c6e9c456"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">avgNumDecodedTokensPerIter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats26avgNumDecodedTokensPerIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Average number of tokens decoded per request per iteration. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStatsE"></span><span id="tensorrt_llm::executor::IterationStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IterationStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of a single iteration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats9timestampE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats9timestampE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats9timestampE"></span><span id="tensorrt_llm::executor::IterationStats::timestamp__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1af1d7e6fb431b3bae3be69d0cc3a6b00e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">timestamp</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats9timestampE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Ending time of this iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats4iterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats4iterE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats4iterE"></span><span id="tensorrt_llm::executor::IterationStats::iter__IterationType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ae19cff294da29d71592cabdc54be9774"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats4iterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Iteration id. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats13iterLatencyMSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats13iterLatencyMSE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats13iterLatencyMSE"></span><span id="tensorrt_llm::executor::IterationStats::iterLatencyMS__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a08d812da19f047a777e11e217b82eaa8"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iterLatencyMS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats13iterLatencyMSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Iteration latency (ms) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats31newActiveRequestsQueueLatencyMSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats31newActiveRequestsQueueLatencyMSE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats31newActiveRequestsQueueLatencyMSE"></span><span id="tensorrt_llm::executor::IterationStats::newActiveRequestsQueueLatencyMS__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ac5d8644864440b8d0208ac6e0946025e"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">newActiveRequestsQueueLatencyMS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats31newActiveRequestsQueueLatencyMSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The total time spent in queue by the requests that became active in this iteration (ms) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats20numNewActiveRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats20numNewActiveRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats20numNewActiveRequestsE"></span><span id="tensorrt_llm::executor::IterationStats::numNewActiveRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ac27bf9ead5c27b5b15d390b57919b966"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numNewActiveRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20numNewActiveRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of new fetched active requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats17numActiveRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats17numActiveRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats17numActiveRequestsE"></span><span id="tensorrt_llm::executor::IterationStats::numActiveRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a7b8b4c8acde99a1eb3de70050e770458"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numActiveRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17numActiveRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of active requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats17numQueuedRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats17numQueuedRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats17numQueuedRequestsE"></span><span id="tensorrt_llm::executor::IterationStats::numQueuedRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1abddd1933dca91cbf6336b10e3fdf6e1b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numQueuedRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17numQueuedRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of queued requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats20numCompletedRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats20numCompletedRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats20numCompletedRequestsE"></span><span id="tensorrt_llm::executor::IterationStats::numCompletedRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ae7a03c208c003d49e477c90b5101c4cc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numCompletedRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20numCompletedRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of requests that were completed in this iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats20maxNumActiveRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats20maxNumActiveRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats20maxNumActiveRequestsE"></span><span id="tensorrt_llm::executor::IterationStats::maxNumActiveRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ac8c5c8bf27f8c0eec9fa7ae41da3fe67"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNumActiveRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20maxNumActiveRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of max active requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats18maxBatchSizeStaticE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats18maxBatchSizeStaticE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats18maxBatchSizeStaticE"></span><span id="tensorrt_llm::executor::IterationStats::maxBatchSizeStatic__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a2916efcce10da5d182bb478c6ab26182"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxBatchSizeStatic</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats18maxBatchSizeStaticE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Static max batch size passed to the executor. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats28maxBatchSizeTunerRecommendedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats28maxBatchSizeTunerRecommendedE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats28maxBatchSizeTunerRecommendedE"></span><span id="tensorrt_llm::executor::IterationStats::maxBatchSizeTunerRecommended__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a13cd7271b5c9263e800f452eb998a1f7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxBatchSizeTunerRecommended</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats28maxBatchSizeTunerRecommendedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Batch size produced by dynamic tuner based on input stats. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats19maxBatchSizeRuntimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats19maxBatchSizeRuntimeE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats19maxBatchSizeRuntimeE"></span><span id="tensorrt_llm::executor::IterationStats::maxBatchSizeRuntime__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a2e21e17176e117804ea1702ba1a44041"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxBatchSizeRuntime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19maxBatchSizeRuntimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>@brife The min of maxBatchSizeStatic and maxBatchSizeRuntimeUpperbound </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats18maxNumTokensStaticE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats18maxNumTokensStaticE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats18maxNumTokensStaticE"></span><span id="tensorrt_llm::executor::IterationStats::maxNumTokensStatic__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1abade518ffdca3b4d0f8199d3174d9785"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNumTokensStatic</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats18maxNumTokensStaticE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>@brife Static max num tokens passed to the executor </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats28maxNumTokensTunerRecommendedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats28maxNumTokensTunerRecommendedE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats28maxNumTokensTunerRecommendedE"></span><span id="tensorrt_llm::executor::IterationStats::maxNumTokensTunerRecommended__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a1e0da1bdf37481742d2edea8f12e223f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNumTokensTunerRecommended</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats28maxNumTokensTunerRecommendedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>@brife Max num tokens produced by dynamic tuner based on input stats </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats19maxNumTokensRuntimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats19maxNumTokensRuntimeE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats19maxNumTokensRuntimeE"></span><span id="tensorrt_llm::executor::IterationStats::maxNumTokensRuntime__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ae24f995f04e83c7135a9add67b135c9b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNumTokensRuntime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19maxNumTokensRuntimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>@brife The runtime max num tokens </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats11gpuMemUsageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats11gpuMemUsageE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats11gpuMemUsageE"></span><span id="tensorrt_llm::executor::IterationStats::gpuMemUsage__s"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ac975025c2e40a167051e4c79afcea908"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpuMemUsage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats11gpuMemUsageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>GPU memory usage in bytes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats11cpuMemUsageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats11cpuMemUsageE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats11cpuMemUsageE"></span><span id="tensorrt_llm::executor::IterationStats::cpuMemUsage__s"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1abba95b563f5fc8459b626b05472eef38"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cpuMemUsage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats11cpuMemUsageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>CPU memory usage in bytes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats14pinnedMemUsageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats14pinnedMemUsageE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats14pinnedMemUsageE"></span><span id="tensorrt_llm::executor::IterationStats::pinnedMemUsage__s"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a99f299d90d3366ae1f694890fd859455"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinnedMemUsage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats14pinnedMemUsageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Pinned memory usage in bytes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats12kvCacheStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats12kvCacheStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats12kvCacheStatsE"></span><span id="tensorrt_llm::executor::IterationStats::kvCacheStats__std::optional:KvCacheStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1ac234045a5fdd46880205b99f6d326977"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="tensorrt_llm::executor::KvCacheStats"><span class="n"><span class="pre">KvCacheStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats12kvCacheStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to KV caches. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats17crossKvCacheStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats17crossKvCacheStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats17crossKvCacheStatsE"></span><span id="tensorrt_llm::executor::IterationStats::crossKvCacheStats__std::optional:KvCacheStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a3f985a83e7ba30a463c13c4cafff87fe"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="tensorrt_llm::executor::KvCacheStats"><span class="n"><span class="pre">KvCacheStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">crossKvCacheStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17crossKvCacheStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to cross KV caches. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats19staticBatchingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats19staticBatchingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats19staticBatchingStatsE"></span><span id="tensorrt_llm::executor::IterationStats::staticBatchingStats__std::optional:StaticBatchingStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a361f059648067ed27ccd1baa91656e8f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE" title="tensorrt_llm::executor::StaticBatchingStats"><span class="n"><span class="pre">StaticBatchingStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">staticBatchingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19staticBatchingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to static batching. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats21inflightBatchingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats21inflightBatchingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats21inflightBatchingStatsE"></span><span id="tensorrt_llm::executor::IterationStats::inflightBatchingStats__std::optional:InflightBatchingStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a8a9f5f9365d9c7cb7e2adcb5e76fb811"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE" title="tensorrt_llm::executor::InflightBatchingStats"><span class="n"><span class="pre">InflightBatchingStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inflightBatchingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats21inflightBatchingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to inflight batching. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14IterationStats17specDecodingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14IterationStats17specDecodingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor14IterationStats17specDecodingStatsE"></span><span id="tensorrt_llm::executor::IterationStats::specDecodingStats__std::optional:SpecDecodingStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1IterationStats_1a5dfb69ab8a08d55901d6011307f5adb0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE" title="tensorrt_llm::executor::SpecDecodingStats"><span class="n"><span class="pre">SpecDecodingStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecodingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17specDecodingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to speculative decoding. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStatsE"></span><span id="tensorrt_llm::executor::KvCacheStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of a KV cache manager. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats12maxNumBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats12maxNumBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats12maxNumBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::maxNumBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a2952c746c5724a096a0f90037cc95a9b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNumBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12maxNumBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Max number of blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats13freeNumBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats13freeNumBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats13freeNumBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::freeNumBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a4393a6d986ba543b215c761888732ddc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">freeNumBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats13freeNumBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of free blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats13usedNumBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats13usedNumBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats13usedNumBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::usedNumBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a9cd563d7a3d9a49a7bea72fbb386fd0c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usedNumBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats13usedNumBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of used blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats14tokensPerBlockE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats14tokensPerBlockE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats14tokensPerBlockE"></span><span id="tensorrt_llm::executor::KvCacheStats::tokensPerBlock__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a7b0c51dae64cd0af386fb0f0104dedc8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokensPerBlock</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats14tokensPerBlockE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of tokens per block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats16allocTotalBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats16allocTotalBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats16allocTotalBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::allocTotalBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a88f44cd389cb3813a93245cab0aa96d8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocTotalBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats16allocTotalBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of total allocated block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats14allocNewBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats14allocNewBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats14allocNewBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::allocNewBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a804a58fa6ad301b7ddfe922cb413d265"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocNewBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats14allocNewBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of newly allocated block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats12reusedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats12reusedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats12reusedBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::reusedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a1c54f95634272634dcb7779bfd87e162"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reusedBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12reusedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of reused block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats12missedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats12missedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats12missedBlocksE"></span><span id="tensorrt_llm::executor::KvCacheStats::missedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1aeb9ea6e480694674dceed68166950f13"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">missedBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12missedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of not reused block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KvCacheStats12cacheHitRateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KvCacheStats12cacheHitRateE"></span><span id="_CPPv2N12tensorrt_llm8executor12KvCacheStats12cacheHitRateE"></span><span id="tensorrt_llm::executor::KvCacheStats::cacheHitRate__float"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheStats_1a85799d137b58b9b8b9020f9b4e9763f6"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheHitRate</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12cacheHitRateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Measuring the KV Cache reuse rate. cacheHitRate = reusedBlocks / (reusedBlocks + missedBlocks). </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestPerfMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of a request. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1af3b6e1baa7b1613f81570f8541ea18ee"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TimePoint</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">time_point</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">steady_clock</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13timingMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13timingMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13timingMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::timingMetrics__TimingMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1a7ca683e5e7da1cf81d52583af9fd24e8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics"><span class="n"><span class="pre">TimingMetrics</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">timingMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13timingMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14kvCacheMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14kvCacheMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14kvCacheMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::kvCacheMetrics__KvCacheMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1a79c3d18a5665397fb4750ee7c434cec2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics"><span class="n"><span class="pre">KvCacheMetrics</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14kvCacheMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics19speculativeDecodingE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics19speculativeDecodingE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics19speculativeDecodingE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::speculativeDecoding__SpeculativeDecodingMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1a70236489521aa00d4ea8ed64a3273466"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics"><span class="n"><span class="pre">SpeculativeDecodingMetrics</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">speculativeDecoding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics19speculativeDecodingE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9firstIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics9firstIterE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics9firstIterE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::firstIter__std::optional:IterationType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1a39d77233c4e9972b4483bf2e479e8a07"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">firstIter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9firstIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>First iteration where the request was processed. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics8lastIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics8lastIterE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics8lastIterE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::lastIter__std::optional:IterationType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1adbb66032d68985474ae89d2289a39785"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastIter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics8lastIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Last iteration where a token was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics4iterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics4iterE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics4iterE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::iter__std::optional:IterationType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1ae690ff374e4407ab54ccea9f3a5c560a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics4iterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Current iteration. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics23numTotalAllocatedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics23numTotalAllocatedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics23numTotalAllocatedBlocksE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics::numTotalAllocatedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics_1abe5d5e7ff95f4ea2df6b41acebf6a1ec"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numTotalAllocatedBlocks</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics23numTotalAllocatedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of total allocated blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics21numNewAllocatedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics21numNewAllocatedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics21numNewAllocatedBlocksE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics::numNewAllocatedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics_1aba5ba413272bc8ee7b1692cc829c46ed"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numNewAllocatedBlocks</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics21numNewAllocatedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of newly allocated blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numReusedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numReusedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numReusedBlocksE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics::numReusedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics_1ac73cc1e473ddc919c5b87c5ed2fccc2c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numReusedBlocks</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numReusedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of reused blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numMissedBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numMissedBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numMissedBlocksE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics::numMissedBlocks__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics_1abebf60af8d299a1ab57e621cda279070"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numMissedBlocks</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numMissedBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of missed blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics14kvCacheHitRateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics14kvCacheHitRateE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics14kvCacheHitRateE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics::kvCacheHitRate__FloatType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1KvCacheMetrics_1ad348365db4232d51890180b46003e61e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheHitRate</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0.f</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics14kvCacheHitRateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>KV Cache Hit Rate, defined as reusedBlocks / (reusedBlocks + missedBlocks) </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1SpeculativeDecodingMetrics"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics14acceptanceRateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics14acceptanceRateE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics14acceptanceRateE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics::acceptanceRate__FloatType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1SpeculativeDecodingMetrics_1aac8f0be29168140006ed9aabf4c758cf"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptanceRate</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0.f</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics14acceptanceRateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Token acceptance rate for speculative decoding requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics24totalAcceptedDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics24totalAcceptedDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics24totalAcceptedDraftTokensE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics::totalAcceptedDraftTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1SpeculativeDecodingMetrics_1a1773f2841bb0a63d3a70e06423c3daea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">totalAcceptedDraftTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics24totalAcceptedDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of accepted draft tokens. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics16totalDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics16totalDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics16totalDraftTokensE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics::totalDraftTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1SpeculativeDecodingMetrics_1aee1dd092f0689a4b7a989e5d17199cdb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">totalDraftTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics16totalDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of draft tokens used in the request. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TimingMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11arrivalTimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11arrivalTimeE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11arrivalTimeE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::arrivalTime__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1abe219c1af2f20d8d2316d3aac6f04565"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">arrivalTime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11arrivalTimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The time when the request arrived. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18firstScheduledTimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18firstScheduledTimeE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18firstScheduledTimeE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::firstScheduledTime__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1a889fa5895662c4c08a86084e868e8a23"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">firstScheduledTime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18firstScheduledTimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The time when the request was first scheduled. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics14firstTokenTimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics14firstTokenTimeE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics14firstTokenTimeE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::firstTokenTime__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1ab1f8b72554c8029fa21adfdd9d4c90f0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">firstTokenTime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics14firstTokenTimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The time when the first token was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics13lastTokenTimeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics13lastTokenTimeE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics13lastTokenTimeE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::lastTokenTime__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1a1a8512ed1362583afbc951fa61af87df"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastTokenTime</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics13lastTokenTimeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The time when the request was finished. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics20kvCacheTransferStartE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics20kvCacheTransferStartE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics20kvCacheTransferStartE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::kvCacheTransferStart__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1add1d41fa479c923306bf6c32b65b5d08"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheTransferStart</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics20kvCacheTransferStartE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Start time of the KV cache transfer for disaggregated serving. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18kvCacheTransferEndE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18kvCacheTransferEndE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18kvCacheTransferEndE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::kvCacheTransferEnd__TimePoint"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1a6c1e9740382c252ab3f819c55fcaf1ac"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE" title="tensorrt_llm::executor::RequestPerfMetrics::TimePoint"><span class="n"><span class="pre">TimePoint</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheTransferEnd</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18kvCacheTransferEndE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>End time of the KV cache transfer for disaggregated serving. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11kvCacheSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11kvCacheSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11kvCacheSizeE"></span><span id="tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics::kvCacheSize__s"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestPerfMetrics_1_1TimingMetrics_1ac80d30cb61b87a4c7508b4eb6e20320f"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11kvCacheSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>KV Cache size transfer for disaggregated serving. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStatsE"></span><span id="tensorrt_llm::executor::RequestStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of a single request. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats2idE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats2idE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats2idE"></span><span id="tensorrt_llm::executor::RequestStats::id__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1ae069f9ba9675130e5c3e3b9129d2aff1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">id</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats2idE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The request id. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats5stageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats5stageE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats5stageE"></span><span id="tensorrt_llm::executor::RequestStats::stage__RequestStage"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a6d35e8a95e6cccaceae38961edd5a354"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE" title="tensorrt_llm::executor::RequestStage"><span class="n"><span class="pre">RequestStage</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats5stageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The current stage the request is in. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats22contextPrefillPositionE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats22contextPrefillPositionE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats22contextPrefillPositionE"></span><span id="tensorrt_llm::executor::RequestStats::contextPrefillPosition__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1ae2372e9fb63311f9f95f77cdf8d09f5b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">contextPrefillPosition</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22contextPrefillPositionE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If using chunked context, the current context prefill position. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats18numGeneratedTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats18numGeneratedTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats18numGeneratedTokensE"></span><span id="tensorrt_llm::executor::RequestStats::numGeneratedTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a18f2badb3c0c81ffee60f7f8e22d2b99"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numGeneratedTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats18numGeneratedTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The number of generated tokens so far. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats26avgNumDecodedTokensPerIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats26avgNumDecodedTokensPerIterE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats26avgNumDecodedTokensPerIterE"></span><span id="tensorrt_llm::executor::RequestStats::avgNumDecodedTokensPerIter__float"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a44573f7d4ab8fce4a931c661929deab1"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">avgNumDecodedTokensPerIter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats26avgNumDecodedTokensPerIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The average number of decoded tokens per iteration. It is >= 1 for speculative decoding. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats9scheduledE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats9scheduledE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats9scheduledE"></span><span id="tensorrt_llm::executor::RequestStats::scheduled__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1ab928aa93fcf7a22e40adfd10a62aee04"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scheduled</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats9scheduledE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Whether the request is scheduled for the current iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats6pausedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats6pausedE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats6pausedE"></span><span id="tensorrt_llm::executor::RequestStats::paused__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a6f74bde53fc8910bad82297f8f741deb"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">paused</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats6pausedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Whether the request is being paused at the current iteration due to lack of resources (KV cache blocks exhaustion for example) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats15disServingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats15disServingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats15disServingStatsE"></span><span id="tensorrt_llm::executor::RequestStats::disServingStats__std::optional:DisServingRequestStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a85783b94d34f9c189e0fc7033704298b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE" title="tensorrt_llm::executor::DisServingRequestStats"><span class="n"><span class="pre">DisServingRequestStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disServingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats15disServingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stats specific to disaggregated serving. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats26allocTotalBlocksPerRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats26allocTotalBlocksPerRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats26allocTotalBlocksPerRequestE"></span><span id="tensorrt_llm::executor::RequestStats::allocTotalBlocksPerRequest__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1ad619ee70e25193236d62493874e97d05"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocTotalBlocksPerRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats26allocTotalBlocksPerRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of total allocated blocks per request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats24allocNewBlocksPerRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats24allocNewBlocksPerRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats24allocNewBlocksPerRequestE"></span><span id="tensorrt_llm::executor::RequestStats::allocNewBlocksPerRequest__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a1eed06ea5d8edb76e9bc5d6430a46f5e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocNewBlocksPerRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats24allocNewBlocksPerRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of newly allocated blocks per request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats22reusedBlocksPerRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats22reusedBlocksPerRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats22reusedBlocksPerRequestE"></span><span id="tensorrt_llm::executor::RequestStats::reusedBlocksPerRequest__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a4ad8bcc4dd5f94ad97f353ca06e3849d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reusedBlocksPerRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22reusedBlocksPerRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of reused blocks per request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats22missedBlocksPerRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats22missedBlocksPerRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats22missedBlocksPerRequestE"></span><span id="tensorrt_llm::executor::RequestStats::missedBlocksPerRequest__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a11599df8fa9ba9f451a05c2ccc1f4a6b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">missedBlocksPerRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22missedBlocksPerRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of missed blocks per request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12RequestStats24kvCacheHitRatePerRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12RequestStats24kvCacheHitRatePerRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor12RequestStats24kvCacheHitRatePerRequestE"></span><span id="tensorrt_llm::executor::RequestStats::kvCacheHitRatePerRequest__FloatType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStats_1a8ef1ccadf2d3c4681e8f7e3b5e4cdee6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheHitRatePerRequest</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12RequestStats24kvCacheHitRatePerRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>KV Cache Hit Rate per request, defined as reusedBlocks / (reusedBlocks + missedBlocks) </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24RequestStatsPerIterationE"></span><span id="_CPPv2N12tensorrt_llm8executor24RequestStatsPerIterationE"></span><span id="tensorrt_llm::executor::RequestStatsPerIteration"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStatsPerIteration"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of all requests in an iteration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration4iterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24RequestStatsPerIteration4iterE"></span><span id="_CPPv2N12tensorrt_llm8executor24RequestStatsPerIteration4iterE"></span><span id="tensorrt_llm::executor::RequestStatsPerIteration::iter__IterationType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStatsPerIteration_1a089fa8d28441ffb3d0f37bd046329ac7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE" title="tensorrt_llm::executor::IterationType"><span class="n"><span class="pre">IterationType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration4iterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The iteration id for these stats. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration12requestStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor24RequestStatsPerIteration12requestStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor24RequestStatsPerIteration12requestStatsE"></span><span id="tensorrt_llm::executor::RequestStatsPerIteration::requestStats__std::vector:RequestStats:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RequestStatsPerIteration_1af9fbfa79e3985aa6f824b1d7edf934d0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="tensorrt_llm::executor::RequestStats"><span class="n"><span class="pre">RequestStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">requestStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration12requestStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The stats of all active requests for this iteration. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStatsE"></span><span id="tensorrt_llm::executor::SpecDecodingStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpecDecodingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds speculative decoding stats. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats14numDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats14numDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats14numDraftTokensE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::numDraftTokens__SizeType64"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1a2ba8cdb4391b566673ac59fbce115cb0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E" title="tensorrt_llm::executor::SizeType64"><span class="n"><span class="pre">SizeType64</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats14numDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of proposed draft tokens for all requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats17numAcceptedTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats17numAcceptedTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats17numAcceptedTokensE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::numAcceptedTokens__SizeType64"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1a82d228d94c33c11b41707c2275eb9157"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E" title="tensorrt_llm::executor::SizeType64"><span class="n"><span class="pre">SizeType64</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numAcceptedTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats17numAcceptedTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of accepted draft tokens for all requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats26numRequestsWithDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats26numRequestsWithDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats26numRequestsWithDraftTokensE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::numRequestsWithDraftTokens__SizeType64"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1af6b842c2bf45eb3c5711fafb45d6b004"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E" title="tensorrt_llm::executor::SizeType64"><span class="n"><span class="pre">SizeType64</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numRequestsWithDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats26numRequestsWithDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of requests with at least one draft token in batch. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats16acceptanceLengthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats16acceptanceLengthE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats16acceptanceLengthE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::acceptanceLength__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1a4581cb18cb060aff99d56a7bc2ace5b0"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptanceLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats16acceptanceLengthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Acceptance length, defined as average number of tokens produced per step for all requests with at least one draft token. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13iterLatencyMSE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats13iterLatencyMSE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats13iterLatencyMSE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::iterLatencyMS__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1abdfdf1844f06c033b41caf259c28859c"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">iterLatencyMS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13iterLatencyMSE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Iteration latency for draft token generation only (ms) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13draftOverheadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17SpecDecodingStats13draftOverheadE"></span><span id="_CPPv2N12tensorrt_llm8executor17SpecDecodingStats13draftOverheadE"></span><span id="tensorrt_llm::executor::SpecDecodingStats::draftOverhead__double"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpecDecodingStats_1a5112514dcef0134ab8e349ab0090553c"></span><span class="kt"><span class="pre">double</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftOverhead</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13draftOverheadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Draft overhead, defined as iterLatencyMS (specdec) / iterLatencyMS (total) </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStatsE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStatsE"></span><span id="tensorrt_llm::executor::StaticBatchingStats"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StaticBatchingStats</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>Struct that holds the stats of static batching models for a single iteration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStats20numScheduledRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStats20numScheduledRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStats20numScheduledRequestsE"></span><span id="tensorrt_llm::executor::StaticBatchingStats::numScheduledRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats_1a85a29e377cc0afc1fa3b5ac5e2426509"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numScheduledRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats20numScheduledRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of scheduled requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStats18numContextRequestsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStats18numContextRequestsE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStats18numContextRequestsE"></span><span id="tensorrt_llm::executor::StaticBatchingStats::numContextRequests__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats_1a44b1d7847625f6042de40ef2a5c8ec70"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numContextRequests</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats18numContextRequestsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of requests in context stage. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numCtxTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStats12numCtxTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStats12numCtxTokensE"></span><span id="tensorrt_llm::executor::StaticBatchingStats::numCtxTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats_1aed7aa230c825577f4acb9d43a6f5176d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numCtxTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numCtxTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of context tokens in the iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numGenTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStats12numGenTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStats12numGenTokensE"></span><span id="tensorrt_llm::executor::StaticBatchingStats::numGenTokens__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats_1a48522c73455172970cb9dc292e53fcd3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numGenTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numGenTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of tokens to generate in the iteration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19StaticBatchingStats13emptyGenSlotsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19StaticBatchingStats13emptyGenSlotsE"></span><span id="_CPPv2N12tensorrt_llm8executor19StaticBatchingStats13emptyGenSlotsE"></span><span id="tensorrt_llm::executor::StaticBatchingStats::emptyGenSlots__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1StaticBatchingStats_1a3316815cfbe07bb2a486de89b7024c41"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">emptyGenSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats13emptyGenSlotsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Total number of unused generation token slots. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0_bEN12tensorrt_llm8executor10TypeTraitsE">
|
||
<span id="_CPPv3I0_bEN12tensorrt_llm8executor10TypeTraitsE"></span><span id="_CPPv2I0_bEN12tensorrt_llm8executor10TypeTraitsE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><a class="headerlink" href="#_CPPv4I0_bEN12tensorrt_llm8executor10TypeTraitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <types.h></em></div>
|
||
<p>For converting a C++ data type to a <code class="docutils literal notranslate"><span class="pre">TrtLmmDataType</span></code>. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIbEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsIbEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsIbEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01bool_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIbEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsIbE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsIbE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsIbE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:b:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01bool_01_4_1a0cf040c47d93165f569715a3e02f22ac"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType5kBOOLE" title="tensorrt_llm::executor::DataType::kBOOL"><span class="n"><span class="pre">kBOOL</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIbE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIfEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsIfEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsIfEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01float_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIfEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsIfE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsIfE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsIfE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:float:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01float_01_4_1aa3c4709453f3b513d302e3b2e843a464"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP32E" title="tensorrt_llm::executor::DataType::kFP32"><span class="n"><span class="pre">kFP32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIfE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsI4halfEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsI4halfEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsI4halfEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01half_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">half</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsI4halfEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsI4halfE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsI4halfE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsI4halfE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:half:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01half_01_4_1a8b0cc854df1f430f8e8eea8f3b8a733b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP16E" title="tensorrt_llm::executor::DataType::kFP16"><span class="n"><span class="pre">kFP16</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsI4halfE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int32_tEEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsINSt7int32_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsINSt7int32_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int32__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int32_tEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int32_tEE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsINSt7int32_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsINSt7int32_tEE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:std::int32_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int32__t_01_4_1a506e37fcee8102b90f320257e12ec485"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT32E" title="tensorrt_llm::executor::DataType::kINT32"><span class="n"><span class="pre">kINT32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int32_tEE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int64_tEEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsINSt7int64_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsINSt7int64_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int64__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int64_tEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int64_tEE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsINSt7int64_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsINSt7int64_tEE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:std::int64_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int64__t_01_4_1ae1e174a531a0abf5c31ed44525da252d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT64E" title="tensorrt_llm::executor::DataType::kINT64"><span class="n"><span class="pre">kINT64</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int64_tEE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt6int8_tEEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsINSt6int8_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsINSt6int8_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int8__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int8_t</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt6int8_tEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt6int8_tEE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsINSt6int8_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsINSt6int8_tEE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:std::int8_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1int8__t_01_4_1aaed76141f49476650f91cfbec3367e13"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType5kINT8E" title="tensorrt_llm::executor::DataType::kINT8"><span class="n"><span class="pre">kINT8</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt6int8_tEE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEEE">
|
||
<span id="_CPPv3IEN12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1uint8__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:std::uint8_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01std_1_1uint8__t_01_4_1a36e64efaf2f35f9e9d97c62fb4e74f84"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType6kUINT8E" title="tensorrt_llm::executor::DataType::kUINT8"><span class="n"><span class="pre">kUINT8</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01T_01_5_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TypeTraits</span></span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE" title="tensorrt_llm::executor::TypeTraits<T*>::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10TypeTraitsIP1TE5valueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10TypeTraitsIP1TE5valueE"></span><span id="_CPPv2N12tensorrt_llm8executor10TypeTraitsIP1TE5valueE"></span><span id="tensorrt_llm::executor::TypeTraits:TP:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1TypeTraits_3_01T_01_5_01_4_1a919c622815440975c53c2ba6f5e02c34"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE" title="tensorrt_llm::executor::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT64E" title="tensorrt_llm::executor::DataType::kINT64"><span class="n"><span class="pre">kINT64</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIP1TE5valueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="executor-h">
|
||
<h2>executor.h<a class="headerlink" href="#executor-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm13batch_managerE">
|
||
<span id="_CPPv3N12tensorrt_llm13batch_managerE"></span><span id="_CPPv2N12tensorrt_llm13batch_managerE"></span><span id="tensorrt_llm::batch_manager"></span><span class="target" id="namespacetensorrt__llm_1_1batch__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batch_manager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE">
|
||
<span id="_CPPv3N12tensorrt_llm13batch_manager16kv_cache_managerE"></span><span id="_CPPv2N12tensorrt_llm13batch_manager16kv_cache_managerE"></span><span id="tensorrt_llm::batch_manager::kv_cache_manager"></span><span class="target" id="namespacetensorrt__llm_1_1batch__manager_1_1kv__cache__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kv_cache_manager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17RetentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17RetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor17RetentionPriorityE"></span><span class="target" id="executor_8h_1a7d47a118ea2835238c34ba65f7ac692e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RetentionPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16KVCacheEventDataE"></span><span id="_CPPv2N12tensorrt_llm8executor16KVCacheEventDataE"></span><span class="target" id="executor_8h_1a2f045efaee75f7c0e1463d79434ad0b8"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEventData</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">variant</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE" title="tensorrt_llm::executor::KVCacheCreatedData"><span class="n"><span class="pre">KVCacheCreatedData</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE" title="tensorrt_llm::executor::KVCacheStoredData"><span class="n"><span class="pre">KVCacheStoredData</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE" title="tensorrt_llm::executor::KVCacheRemovedData"><span class="n"><span class="pre">KVCacheRemovedData</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7versionEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7versionEv"></span><span id="_CPPv2N12tensorrt_llm8executor7versionEv"></span><span id="tensorrt_llm::executor::version"></span><span class="target" id="executor_8h_1ada83e7ba49c4897a21cdb8706c97fce1"></span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">version</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7versionEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Version of TRT-LLM. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21AdditionalModelOutputE"></span><span id="_CPPv2N12tensorrt_llm8executor21AdditionalModelOutputE"></span><span id="tensorrt_llm::executor::AdditionalModelOutput"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1AdditionalModelOutput"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalModelOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Additional output that should be gathered. </p>
|
||
<p>By default gather output of shape [beamWidth, x] from each generation phase. If gatherContext is true, also gather output of shape [promptLen, x] from context phase. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput21AdditionalModelOutputENSt6stringEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21AdditionalModelOutput21AdditionalModelOutputENSt6stringEb"></span><span id="_CPPv2N12tensorrt_llm8executor21AdditionalModelOutput21AdditionalModelOutputENSt6stringEb"></span><span id="tensorrt_llm::executor::AdditionalModelOutput::AdditionalModelOutput__ss.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1AdditionalModelOutput_1aaabca78a393a611264fbe26c731ce9a7"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalModelOutput</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">gatherContext</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput21AdditionalModelOutputENSt6stringEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor21AdditionalModelOutputeqERK21AdditionalModelOutput">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor21AdditionalModelOutputeqERK21AdditionalModelOutput"></span><span id="_CPPv2NK12tensorrt_llm8executor21AdditionalModelOutputeqERK21AdditionalModelOutput"></span><span id="tensorrt_llm::executor::AdditionalModelOutput::eq-operator__AdditionalModelOutputCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1AdditionalModelOutput_1ac21857dd04183a43290f9f68115b5dc8"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor21AdditionalModelOutputeqERK21AdditionalModelOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput4nameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21AdditionalModelOutput4nameE"></span><span id="_CPPv2N12tensorrt_llm8executor21AdditionalModelOutput4nameE"></span><span id="tensorrt_llm::executor::AdditionalModelOutput::name__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1AdditionalModelOutput_1a3e127e5e2fc7bba06343d5230f4dc01d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput4nameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput13gatherContextE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor21AdditionalModelOutput13gatherContextE"></span><span id="_CPPv2N12tensorrt_llm8executor21AdditionalModelOutput13gatherContextE"></span><span id="tensorrt_llm::executor::AdditionalModelOutput::gatherContext__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1AdditionalModelOutput_1a2d62df18f67e22c1b9fed4893c539d78"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gatherContext</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput13gatherContextE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutputE"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutputE"></span><span id="tensorrt_llm::executor::AdditionalOutput"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputENSt6stringE6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputENSt6stringE6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputENSt6stringE6Tensor"></span><span id="tensorrt_llm::executor::AdditionalOutput::AdditionalOutput__ss.Tensor"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a82544acd3b20d5e498e9f7299a8f927c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalOutput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">name</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">output</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputENSt6stringE6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput"></span><span id="tensorrt_llm::executor::AdditionalOutput::AdditionalOutput__AdditionalOutputCR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a5f5d32989a819a49298c5809a3d911b2"></span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalOutput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput" title="tensorrt_llm::executor::AdditionalOutput::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput"></span><span id="tensorrt_llm::executor::AdditionalOutput::AdditionalOutput__AdditionalOutputRR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a532eaea55185fb8f591c19c0f5688319"></span><span class="sig-name descname"><span class="n"><span class="pre">AdditionalOutput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput" title="tensorrt_llm::executor::AdditionalOutput::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERK16AdditionalOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutputaSERK16AdditionalOutput"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutputaSERK16AdditionalOutput"></span><span id="tensorrt_llm::executor::AdditionalOutput::assign-operator__AdditionalOutputCR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a6c610373aa07ba4e1c15741fcecf11f1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERK16AdditionalOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERR16AdditionalOutput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutputaSERR16AdditionalOutput"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutputaSERR16AdditionalOutput"></span><span id="tensorrt_llm::executor::AdditionalOutput::assign-operator__AdditionalOutputRR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a77d75626d7317a05426df47088d56722"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERR16AdditionalOutput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutputD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutputD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutputD0Ev"></span><span id="tensorrt_llm::executor::AdditionalOutput::~AdditionalOutput"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a0683bf462075ecfcf491eb591662fa02"></span><span class="sig-name descname"><span class="n"><span class="pre">~AdditionalOutput</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutput4nameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutput4nameE"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutput4nameE"></span><span id="tensorrt_llm::executor::AdditionalOutput::name__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a36fb3be58891af0d091c8c097f1314b0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput4nameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16AdditionalOutput6outputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16AdditionalOutput6outputE"></span><span id="_CPPv2N12tensorrt_llm8executor16AdditionalOutput6outputE"></span><span id="tensorrt_llm::executor::AdditionalOutput::output__Tensor"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1AdditionalOutput_1a747114548ce6f99cea8d24c64fc5b98d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">output</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput6outputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfigE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a608a373e86f6f3ffe4c31afa765a7c4b"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BackendType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType7DEFAULTE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType7DEFAULTE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType7DEFAULTE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a608a373e86f6f3ffe4c31afa765a7c4ba5b39c8b553c821e7cddc6da64b5bd2ee"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DEFAULT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType7DEFAULTE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3MPIE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3MPIE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3MPIE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a608a373e86f6f3ffe4c31afa765a7c4ba3b124f211648678733ce66a1b361f06c"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MPI</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3MPIE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3UCXE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3UCXE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3UCXE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a608a373e86f6f3ffe4c31afa765a7c4baa3775004103537c1084b63811ff97b7e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UCX</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3UCXE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType4NIXLE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType4NIXLE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType4NIXLE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a608a373e86f6f3ffe4c31afa765a7c4ba7e5550ecbb056bfa25fd413b9f11e827"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">NIXL</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType4NIXLE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22CacheTransceiverConfigENSt8optionalI11BackendTypeEENSt8optionalI6size_tEENSt8optionalIiEENSt8optionalIiEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig22CacheTransceiverConfigENSt8optionalI11BackendTypeEENSt8optionalI6size_tEENSt8optionalIiEENSt8optionalIiEE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig22CacheTransceiverConfigENSt8optionalI11BackendTypeEENSt8optionalI6size_tEENSt8optionalIiEENSt8optionalIiEE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::CacheTransceiverConfig__std::optional:BackendType:.std::optional:s:.std::optional:i:.std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a8a50c689ed554d8dad7c781115310af4"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE" title="tensorrt_llm::executor::CacheTransceiverConfig::BackendType"><span class="n"><span class="pre">BackendType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">backendType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvTransferTimeoutMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvTransferSenderFutureTimeoutMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22CacheTransceiverConfigENSt8optionalI11BackendTypeEENSt8optionalI6size_tEENSt8optionalIiEENSt8optionalIiEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfigeqERK22CacheTransceiverConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22CacheTransceiverConfigeqERK22CacheTransceiverConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor22CacheTransceiverConfigeqERK22CacheTransceiverConfig"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::eq-operator__CacheTransceiverConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a336895c26e12f158783c74cfbb0e63bd"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfigeqERK22CacheTransceiverConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig14setBackendTypeENSt8optionalI11BackendTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig14setBackendTypeENSt8optionalI11BackendTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig14setBackendTypeENSt8optionalI11BackendTypeEE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::setBackendType__std::optional:BackendType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a12a448a03e66092199c42839a7403d3d"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBackendType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE" title="tensorrt_llm::executor::CacheTransceiverConfig::BackendType"><span class="n"><span class="pre">BackendType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">backendType</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig14setBackendTypeENSt8optionalI11BackendTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20setMaxTokensInBufferENSt8optionalI6size_tEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig20setMaxTokensInBufferENSt8optionalI6size_tEE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig20setMaxTokensInBufferENSt8optionalI6size_tEE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::setMaxTokensInBuffer__std::optional:s:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1af93ad0d5fb5b4b38d9d3fa04fa2d456f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxTokensInBuffer</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokensInBuffer</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20setMaxTokensInBufferENSt8optionalI6size_tEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22setKvTransferTimeoutMsENSt8optionalIiEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig22setKvTransferTimeoutMsENSt8optionalIiEE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig22setKvTransferTimeoutMsENSt8optionalIiEE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::setKvTransferTimeoutMs__std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a12b3b58d7e2d7da4212c421aad370398"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setKvTransferTimeoutMs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvTransferTimeoutMs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22setKvTransferTimeoutMsENSt8optionalIiEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig34setKvTransferSenderFutureTimeoutMsENSt8optionalIiEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig34setKvTransferSenderFutureTimeoutMsENSt8optionalIiEE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig34setKvTransferSenderFutureTimeoutMsENSt8optionalIiEE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::setKvTransferSenderFutureTimeoutMs__std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a8214f08f61a819e173d5b2f8c16c57a7"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setKvTransferSenderFutureTimeoutMs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvTransferSenderFutureTimeoutMs</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig34setKvTransferSenderFutureTimeoutMsENSt8optionalIiEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig20getMaxTokensInBufferEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22CacheTransceiverConfig20getMaxTokensInBufferEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22CacheTransceiverConfig20getMaxTokensInBufferEv"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::getMaxTokensInBufferC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a4b14a01d128996da784a03f9014093a7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxTokensInBuffer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig20getMaxTokensInBufferEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig14getBackendTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22CacheTransceiverConfig14getBackendTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22CacheTransceiverConfig14getBackendTypeEv"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::getBackendTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1addaa1d6de0de3f54df535717224fdc87"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE" title="tensorrt_llm::executor::CacheTransceiverConfig::BackendType"><span class="n"><span class="pre">BackendType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBackendType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig14getBackendTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig22getKvTransferTimeoutMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22CacheTransceiverConfig22getKvTransferTimeoutMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22CacheTransceiverConfig22getKvTransferTimeoutMsEv"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::getKvTransferTimeoutMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a9ec96a0513dc03153ac8641176cec3c7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKvTransferTimeoutMs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig22getKvTransferTimeoutMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig34getKvTransferSenderFutureTimeoutMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22CacheTransceiverConfig34getKvTransferSenderFutureTimeoutMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22CacheTransceiverConfig34getKvTransferSenderFutureTimeoutMsEv"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::getKvTransferSenderFutureTimeoutMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a086ae3cf227271c65c2e411ce57412da"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKvTransferSenderFutureTimeoutMs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig34getKvTransferSenderFutureTimeoutMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig12mBackendTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig12mBackendTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig12mBackendTypeE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::mBackendType__std::optional:BackendType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1ae8dba4f5770dfa85a3bda71563b17acd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE" title="tensorrt_llm::executor::CacheTransceiverConfig::BackendType"><span class="n"><span class="pre">BackendType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBackendType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig12mBackendTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig18mMaxTokensInBufferE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig18mMaxTokensInBufferE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig18mMaxTokensInBufferE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::mMaxTokensInBuffer__std::optional:s:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1adba9ef50ba40d211880dbeb031eb707e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxTokensInBuffer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig18mMaxTokensInBufferE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The maximum number of tokens that the CacheTransceiver’s pre-allocated buffer can hold. If the number of kvCache tokens to be transferred for a single request is greater than this value, the performance of the cache transfer may be degraded. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20mKvTransferTimeoutMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig20mKvTransferTimeoutMsE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig20mKvTransferTimeoutMsE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::mKvTransferTimeoutMs__std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1aa3f038830e26b81ae091da685b2d9f1f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mKvTransferTimeoutMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20mKvTransferTimeoutMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig32mKvTransferSenderFutureTimeoutMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22CacheTransceiverConfig32mKvTransferSenderFutureTimeoutMsE"></span><span id="_CPPv2N12tensorrt_llm8executor22CacheTransceiverConfig32mKvTransferSenderFutureTimeoutMsE"></span><span id="tensorrt_llm::executor::CacheTransceiverConfig::mKvTransferSenderFutureTimeoutMs__std::optional:i:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1CacheTransceiverConfig_1a0a06b3ce7e520f77419d0c7aa4e6de1d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mKvTransferSenderFutureTimeoutMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig32mKvTransferSenderFutureTimeoutMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParamsE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParamsE"></span><span id="tensorrt_llm::executor::ContextPhaseParams"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1aa81094bd70b775d6648abf5716a9251b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeNSt8optionalI9VecTokensEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeNSt8optionalI9VecTokensEE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeNSt8optionalI9VecTokensEE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams__VecTokens.RequestIdType.std::optional:VecTokens:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a7a3c7d2c86ed63e2490f5f812b3a70eb"></span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">firstGenTokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="tensorrt_llm::executor::ContextPhaseParams::RequestIdType"><span class="n"><span class="pre">RequestIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">reqId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">draftTokens</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeNSt8optionalI9VecTokensEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypePvNSt8optionalI9VecTokensEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypePvNSt8optionalI9VecTokensEE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypePvNSt8optionalI9VecTokensEE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams__VecTokens.RequestIdType.voidP.std::optional:VecTokens:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a3503c7d6ce6da512c38ff7be70501ff6"></span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">firstGenTokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="tensorrt_llm::executor::ContextPhaseParams::RequestIdType"><span class="n"><span class="pre">RequestIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">reqId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">draftTokens</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypePvNSt8optionalI9VecTokensEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeRKNSt6vectorIcEENSt8optionalI9VecTokensEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeRKNSt6vectorIcEENSt8optionalI9VecTokensEE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeRKNSt6vectorIcEENSt8optionalI9VecTokensEE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams__VecTokens.RequestIdType.std::vector:c:CR.std::optional:VecTokens:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1adc974467edc56a67e087d40e344cfe9f"></span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">firstGenTokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="tensorrt_llm::executor::ContextPhaseParams::RequestIdType"><span class="n"><span class="pre">RequestIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">reqId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">serializedState</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">draftTokens</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeRKNSt6vectorIcEENSt8optionalI9VecTokensEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams"></span><span id="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams__ContextPhaseParamsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a6656cde607b9fbe8c3810cce3364d69a"></span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams" title="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams"></span><span id="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams__ContextPhaseParamsRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1aceb361f1e38c515573ff87c0ce8b12c1"></span><span class="sig-name descname"><span class="n"><span class="pre">ContextPhaseParams</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams" title="tensorrt_llm::executor::ContextPhaseParams::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERK18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParamsaSERK18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParamsaSERK18ContextPhaseParams"></span><span id="tensorrt_llm::executor::ContextPhaseParams::assign-operator__ContextPhaseParamsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a8e835e9f16fd46d1caeff80f8720f634"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERK18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERR18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParamsaSERR18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParamsaSERR18ContextPhaseParams"></span><span id="tensorrt_llm::executor::ContextPhaseParams::assign-operator__ContextPhaseParamsRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a0d6ef185c183fb7d2ceeabcaf7ec0ac3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERR18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParamsD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParamsD0Ev"></span><span id="tensorrt_llm::executor::ContextPhaseParams::~ContextPhaseParams"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a8ea9d3c141f9bbcb9639c8e1a630c57e"></span><span class="sig-name descname"><span class="n"><span class="pre">~ContextPhaseParams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18ContextPhaseParamseqERK18ContextPhaseParams">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18ContextPhaseParamseqERK18ContextPhaseParams"></span><span id="_CPPv2NK12tensorrt_llm8executor18ContextPhaseParamseqERK18ContextPhaseParams"></span><span id="tensorrt_llm::executor::ContextPhaseParams::eq-operator__ContextPhaseParamsCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a54426f8d39c0eab37287f3b313199caf"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParamseqERK18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams17getFirstGenTokensEv">
|
||
<span id="_CPPv3NKR12tensorrt_llm8executor18ContextPhaseParams17getFirstGenTokensEv"></span><span id="_CPPv2NKR12tensorrt_llm8executor18ContextPhaseParams17getFirstGenTokensEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getFirstGenTokensCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1acb2699e4cc7cb332c947c0827159b136"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getFirstGenTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams17getFirstGenTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams14getDraftTokensEv">
|
||
<span id="_CPPv3NKR12tensorrt_llm8executor18ContextPhaseParams14getDraftTokensEv"></span><span id="_CPPv2NKR12tensorrt_llm8executor18ContextPhaseParams14getDraftTokensEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getDraftTokensCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a4e4ab7b9786bd87bc3899380686a7610"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams14getDraftTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NO12tensorrt_llm8executor18ContextPhaseParams17popFirstGenTokensEv">
|
||
<span id="_CPPv3NO12tensorrt_llm8executor18ContextPhaseParams17popFirstGenTokensEv"></span><span id="_CPPv2NO12tensorrt_llm8executor18ContextPhaseParams17popFirstGenTokensEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::popFirstGenTokensO"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a71706e399ff075f0bea621300d1912d1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">popFirstGenTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">&&</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NO12tensorrt_llm8executor18ContextPhaseParams17popFirstGenTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getReqIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18ContextPhaseParams8getReqIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18ContextPhaseParams8getReqIdEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getReqIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1aad7f988239ce101c7bada72d204e8856"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="tensorrt_llm::executor::ContextPhaseParams::RequestIdType"><span class="n"><span class="pre">RequestIdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getReqId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getReqIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18ContextPhaseParams8getStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18ContextPhaseParams8getStateEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a66f6bf5b5bc42e4e0798e54e763e73ff"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8getStateEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams8getStateEv"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams8getStateEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a80aa3ce0559fda9604398603a1153599"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8getStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12releaseStateEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams12releaseStateEv"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams12releaseStateEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::releaseState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1acb798adfd6c9643be6972307932e4959"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">releaseState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12releaseStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams18getSerializedStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18ContextPhaseParams18getSerializedStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18ContextPhaseParams18getSerializedStateEv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::getSerializedStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a96fea647f2bd67f4cea35bb71f82fdf4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">char</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSerializedState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams18getSerializedStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1abb27654f82b7e3719f1383723bddbf49"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StatePtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">decltype</span></span><span class="p"><span class="pre">(</span></span><span class="o"><span class="pre">&</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv" title="tensorrt_llm::executor::ContextPhaseParams::deleter"><span class="n"><span class="pre">deleter</span></span></a><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mReqIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams6mReqIdE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams6mReqIdE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::mReqId__RequestIdType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a948c936825dd49d70b437812c17ba03d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE" title="tensorrt_llm::executor::ContextPhaseParams::RequestIdType"><span class="n"><span class="pre">RequestIdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mReqId</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mReqIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>This request corresponds to the request ID in the context phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams15mFirstGenTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams15mFirstGenTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams15mFirstGenTokensE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::mFirstGenTokens__VecTokens"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a05b2f569fee8bb174eb7a043bbeabb4d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFirstGenTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams15mFirstGenTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The first tokens generated by context executor. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams6mStateE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams6mStateE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::mState__StatePtr"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a3163863c98649154b4af531d0caad5f6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE" title="tensorrt_llm::executor::ContextPhaseParams::StatePtr"><span class="n"><span class="pre">StatePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mState</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">nullptr</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv" title="tensorrt_llm::executor::ContextPhaseParams::deleter"><span class="n"><span class="pre">deleter</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Context phase state of this request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12mDraftTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams12mDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams12mDraftTokensE"></span><span id="tensorrt_llm::executor::ContextPhaseParams::mDraftTokens__std::optional:VecTokens:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1ad4d90613e5f2fd7e2bf0f4565e6bd945"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12mDraftTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The draft tokens generated by context executor. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv"></span><span id="_CPPv2N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv"></span><span id="tensorrt_llm::executor::ContextPhaseParams::deleter__voidCP"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ContextPhaseParams_1a0d07065f40fffbdb98ec8d2aa350a739"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deleter</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfigE"></span><span id="tensorrt_llm::executor::DebugConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DebugConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for debugging output. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig11DebugConfigEbb9StringVec10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig11DebugConfigEbb9StringVec10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig11DebugConfigEbb9StringVec10SizeType32"></span><span id="tensorrt_llm::executor::DebugConfig::DebugConfig__b.b.StringVec.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a441df9531cf182fd1378ecbb0d92ecbc"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DebugConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">debugInputTensors</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">debugOutputTensors</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE" title="tensorrt_llm::executor::DebugConfig::StringVec"><span class="n"><span class="pre">StringVec</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">debugTensorNames</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">debugTensorsMaxIterations</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig11DebugConfigEbb9StringVec10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11DebugConfigeqERK11DebugConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11DebugConfigeqERK11DebugConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor11DebugConfigeqERK11DebugConfig"></span><span id="tensorrt_llm::executor::DebugConfig::eq-operator__DebugConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a8d9d2b7dd7685fb438931bc12a79c555"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfigeqERK11DebugConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11DebugConfig20getDebugInputTensorsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11DebugConfig20getDebugInputTensorsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11DebugConfig20getDebugInputTensorsEv"></span><span id="tensorrt_llm::executor::DebugConfig::getDebugInputTensorsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1aff4941a872c0fb6025fe5c647a93d7ac"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDebugInputTensors</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig20getDebugInputTensorsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11DebugConfig21getDebugOutputTensorsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11DebugConfig21getDebugOutputTensorsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11DebugConfig21getDebugOutputTensorsEv"></span><span id="tensorrt_llm::executor::DebugConfig::getDebugOutputTensorsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1afa176f8fd81f1f1f0d458880ceed890c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDebugOutputTensors</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig21getDebugOutputTensorsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11DebugConfig19getDebugTensorNamesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11DebugConfig19getDebugTensorNamesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11DebugConfig19getDebugTensorNamesEv"></span><span id="tensorrt_llm::executor::DebugConfig::getDebugTensorNamesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1ae3abfb3f9125c4771c012ddfa5e6343f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE" title="tensorrt_llm::executor::DebugConfig::StringVec"><span class="n"><span class="pre">StringVec</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDebugTensorNames</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig19getDebugTensorNamesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11DebugConfig28getDebugTensorsMaxIterationsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11DebugConfig28getDebugTensorsMaxIterationsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11DebugConfig28getDebugTensorsMaxIterationsEv"></span><span id="tensorrt_llm::executor::DebugConfig::getDebugTensorsMaxIterationsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1aa88d2b27e0fe9a82d79fed827a813a40"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDebugTensorsMaxIterations</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig28getDebugTensorsMaxIterationsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig20setDebugInputTensorsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig20setDebugInputTensorsEb"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig20setDebugInputTensorsEb"></span><span id="tensorrt_llm::executor::DebugConfig::setDebugInputTensors__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a89b3498cf45b85b70f71cd1ee9100ea1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDebugInputTensors</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">debugInputTensors</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig20setDebugInputTensorsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig21setDebugOutputTensorsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig21setDebugOutputTensorsEb"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig21setDebugOutputTensorsEb"></span><span id="tensorrt_llm::executor::DebugConfig::setDebugOutputTensors__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a965f04a1e84947f62135f8bed265cef1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDebugOutputTensors</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">debugOutputTensors</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig21setDebugOutputTensorsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig19setDebugTensorNamesERK9StringVec">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig19setDebugTensorNamesERK9StringVec"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig19setDebugTensorNamesERK9StringVec"></span><span id="tensorrt_llm::executor::DebugConfig::setDebugTensorNames__StringVecCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a9f600e7172ec670e0cbe1e45f38b7148"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDebugTensorNames</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE" title="tensorrt_llm::executor::DebugConfig::StringVec"><span class="n"><span class="pre">StringVec</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">debugTensorNames</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig19setDebugTensorNamesERK9StringVec" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig28setDebugTensorsMaxIterationsE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig28setDebugTensorsMaxIterationsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig28setDebugTensorsMaxIterationsE10SizeType32"></span><span id="tensorrt_llm::executor::DebugConfig::setDebugTensorsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1aa15dcb9f178e13549213315946c23e59"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDebugTensorsMaxIterations</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">debugTensorsMaxIterations</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig28setDebugTensorsMaxIterationsE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig9StringVecE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig9StringVecE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1a477b51070633ac6ddff496518f5b0b59"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StringVec</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig18mDebugInputTensorsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig18mDebugInputTensorsE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig18mDebugInputTensorsE"></span><span id="tensorrt_llm::executor::DebugConfig::mDebugInputTensors__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1aa03ddb48ba29e462a2c0f1fc74d7c2a7"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDebugInputTensors</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig18mDebugInputTensorsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If true, debug all input tensors. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig19mDebugOutputTensorsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig19mDebugOutputTensorsE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig19mDebugOutputTensorsE"></span><span id="tensorrt_llm::executor::DebugConfig::mDebugOutputTensors__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1ad85f336e493ae2a557a41370333182cf"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDebugOutputTensors</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig19mDebugOutputTensorsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If true, debug all output tensors. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig17mDebugTensorNamesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig17mDebugTensorNamesE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig17mDebugTensorNamesE"></span><span id="tensorrt_llm::executor::DebugConfig::mDebugTensorNames__StringVec"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1ae88247e3712e572334bcfef248b8d3e4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE" title="tensorrt_llm::executor::DebugConfig::StringVec"><span class="n"><span class="pre">StringVec</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDebugTensorNames</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig17mDebugTensorNamesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If not empty, only debug tensors in this list. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11DebugConfig26mDebugTensorsMaxIterationsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11DebugConfig26mDebugTensorsMaxIterationsE"></span><span id="_CPPv2N12tensorrt_llm8executor11DebugConfig26mDebugTensorsMaxIterationsE"></span><span id="tensorrt_llm::executor::DebugConfig::mDebugTensorsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DebugConfig_1abea452f5c201eb66e378f02960163f88"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDebugTensorsMaxIterations</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig26mDebugTensorsMaxIterationsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If > 0, provide debug tensors for at most debugTensorsMaxIterations past iterations, else dump them to files. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfigE"></span><span id="tensorrt_llm::executor::DecodingConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for the decoding. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig14DecodingConfigENSt8optionalI12DecodingModeEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI13MedusaChoicesEENSt8optionalI11EagleConfigEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig14DecodingConfigENSt8optionalI12DecodingModeEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI13MedusaChoicesEENSt8optionalI11EagleConfigEE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig14DecodingConfigENSt8optionalI12DecodingModeEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI13MedusaChoicesEENSt8optionalI11EagleConfigEE"></span><span id="tensorrt_llm::executor::DecodingConfig::DecodingConfig__std::optional:DecodingMode:.std::optional:LookaheadDecodingConfig:.std::optional:MedusaChoices:.std::optional:EagleConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1af6d3ff1a4fcb6a2114d587e055963e95"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">decodingMode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">lookaheadDecodingConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE" title="tensorrt_llm::executor::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">medusaChoices</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">eagleConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14DecodingConfigENSt8optionalI12DecodingModeEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI13MedusaChoicesEENSt8optionalI11EagleConfigEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfigeqERK14DecodingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfigeqERK14DecodingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfigeqERK14DecodingConfig"></span><span id="tensorrt_llm::executor::DecodingConfig::eq-operator__DecodingConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a72aae341b22cfafe152513cf5570c39f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfigeqERK14DecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig15setDecodingModeERK12DecodingMode">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig15setDecodingModeERK12DecodingMode"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig15setDecodingModeERK12DecodingMode"></span><span id="tensorrt_llm::executor::DecodingConfig::setDecodingMode__DecodingModeCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a67e4a10d6f2bc039bd1a22d86a54b13b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDecodingMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig15setDecodingModeERK12DecodingMode" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Sets decoding mode. Some modes require the use of their own setters. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfig15getDecodingModeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfig15getDecodingModeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfig15getDecodingModeEv"></span><span id="tensorrt_llm::executor::DecodingConfig::getDecodingModeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1ad949398a15d2b85e99b569783243d069"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDecodingMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig15getDecodingModeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig26setLookaheadDecodingConfigERK23LookaheadDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig26setLookaheadDecodingConfigERK23LookaheadDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig26setLookaheadDecodingConfigERK23LookaheadDecodingConfig"></span><span id="tensorrt_llm::executor::DecodingConfig::setLookaheadDecodingConfig__LookaheadDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a9242afd290fb8532151e29cb56cf1513"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLookaheadDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lookaheadDecodingConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig26setLookaheadDecodingConfigERK23LookaheadDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Sets lookahead decoding mode and config. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig31enableSeamlessLookaheadDecodingEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig31enableSeamlessLookaheadDecodingEv"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig31enableSeamlessLookaheadDecodingEv"></span><span id="tensorrt_llm::executor::DecodingConfig::enableSeamlessLookaheadDecoding"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a001ce705646ede801571938f12b2605d"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enableSeamlessLookaheadDecoding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig31enableSeamlessLookaheadDecodingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfig26getLookaheadDecodingConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfig26getLookaheadDecodingConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfig26getLookaheadDecodingConfigEv"></span><span id="tensorrt_llm::executor::DecodingConfig::getLookaheadDecodingConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a50e1dff2897795cf6a4bd5f0a4cc05b3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLookaheadDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig26getLookaheadDecodingConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfig33getLookaheadDecodingMaxNumRequestEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfig33getLookaheadDecodingMaxNumRequestEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfig33getLookaheadDecodingMaxNumRequestEv"></span><span id="tensorrt_llm::executor::DecodingConfig::getLookaheadDecodingMaxNumRequestC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a672f0e8e633e5f6e3df1ff79ba025746"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLookaheadDecodingMaxNumRequest</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig33getLookaheadDecodingMaxNumRequestEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig16setMedusaChoicesERK13MedusaChoices">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig16setMedusaChoicesERK13MedusaChoices"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig16setMedusaChoicesERK13MedusaChoices"></span><span id="tensorrt_llm::executor::DecodingConfig::setMedusaChoices__MedusaChoicesCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1aa970dcb3b1ce45250e4f33febd6f20e8"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMedusaChoices</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE" title="tensorrt_llm::executor::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig16setMedusaChoicesERK13MedusaChoices" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Sets medusa mode and config. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfig16getMedusaChoicesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfig16getMedusaChoicesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfig16getMedusaChoicesEv"></span><span id="tensorrt_llm::executor::DecodingConfig::getMedusaChoicesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1afd3a11ae26135ac88540c5590de4e2f5"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE" title="tensorrt_llm::executor::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMedusaChoices</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig16getMedusaChoicesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig14setEagleConfigERK11EagleConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig14setEagleConfigERK11EagleConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig14setEagleConfigERK11EagleConfig"></span><span id="tensorrt_llm::executor::DecodingConfig::setEagleConfig__EagleConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a816f97fda3245071c255534ee020a47f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEagleConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14setEagleConfigERK11EagleConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Sets eagle mode and config. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14DecodingConfig14getEagleConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14DecodingConfig14getEagleConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14DecodingConfig14getEagleConfigEv"></span><span id="tensorrt_llm::executor::DecodingConfig::getEagleConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a019c2bcf78f01730757d50af7ba9ba08"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEagleConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig14getEagleConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig13mDecodingModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig13mDecodingModeE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig13mDecodingModeE"></span><span id="tensorrt_llm::executor::DecodingConfig::mDecodingMode__std::optional:DecodingMode:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a44b33dcb361289657af45112b7d7ef65"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodingMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig13mDecodingModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig24mLookaheadDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig24mLookaheadDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig24mLookaheadDecodingConfigE"></span><span id="tensorrt_llm::executor::DecodingConfig::mLookaheadDecodingConfig__std::optional:LookaheadDecodingConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a714a527e462727b8aba4c8ff4e18c35e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLookaheadDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig24mLookaheadDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig14mMedusaChoicesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig14mMedusaChoicesE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig14mMedusaChoicesE"></span><span id="tensorrt_llm::executor::DecodingConfig::mMedusaChoices__std::optional:MedusaChoices:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1a13bb97024510b32cbb56b1606742e7ec"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE" title="tensorrt_llm::executor::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMedusaChoices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14mMedusaChoicesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig12mEagleConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig12mEagleConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig12mEagleConfigE"></span><span id="tensorrt_llm::executor::DecodingConfig::mEagleConfig__std::optional:EagleConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1ac03e1f2278412e56fddd447cde2bae56"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEagleConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig12mEagleConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-attributes">Private Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14DecodingConfig31mLookaheadDecodingMaxNumRequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14DecodingConfig31mLookaheadDecodingMaxNumRequestE"></span><span id="_CPPv2N12tensorrt_llm8executor14DecodingConfig31mLookaheadDecodingMaxNumRequestE"></span><span id="tensorrt_llm::executor::DecodingConfig::mLookaheadDecodingMaxNumRequest__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DecodingConfig_1aa4c9a141818ce59f248eff155b8bb516"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLookaheadDecodingMaxNumRequest</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig31mLookaheadDecodingMaxNumRequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfigE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DynamicBatchConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for dynamic tuning of batch size and max num tokens. During runtime the statistics of input and output lengths are recoreded. Based on these statistics, the batch size and max num tokens are tuned dynamically to better serve the requests. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig18DynamicBatchConfigEbb10SizeType32NSt6vectorINSt4pairI10SizeType3210SizeType32EEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig18DynamicBatchConfigEbb10SizeType32NSt6vectorINSt4pairI10SizeType3210SizeType32EEEE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig18DynamicBatchConfigEbb10SizeType32NSt6vectorINSt4pairI10SizeType3210SizeType32EEEE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::DynamicBatchConfig__b.b.SizeType32.std::vector:std::pair:SizeType32.SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a5a375564bc43dafa4335a84eb8efbe41"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DynamicBatchConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBatchSizeTuning</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableMaxNumTokensTuning</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dynamicBatchMovingAverageWindow</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE" title="tensorrt_llm::executor::DynamicBatchConfig::kDefaultDynamicBatchMovingAverageWindow"><span class="n"><span class="pre">kDefaultDynamicBatchMovingAverageWindow</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSizeTable</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE" title="tensorrt_llm::executor::DynamicBatchConfig::kDefaultBatchSizeTable"><span class="n"><span class="pre">kDefaultBatchSizeTable</span></span></a></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig18DynamicBatchConfigEbb10SizeType32NSt6vectorINSt4pairI10SizeType3210SizeType32EEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig34getDynamicBatchMovingAverageWindowEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18DynamicBatchConfig34getDynamicBatchMovingAverageWindowEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18DynamicBatchConfig34getDynamicBatchMovingAverageWindowEv"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::getDynamicBatchMovingAverageWindowC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1ace3684b897f4e1918c7c16f7dbb0f567"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDynamicBatchMovingAverageWindow</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig34getDynamicBatchMovingAverageWindowEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig24getEnableBatchSizeTuningEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18DynamicBatchConfig24getEnableBatchSizeTuningEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18DynamicBatchConfig24getEnableBatchSizeTuningEv"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::getEnableBatchSizeTuningC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1ae1333e1c22722d1c6f6873bbf8723b73"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableBatchSizeTuning</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig24getEnableBatchSizeTuningEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig27getEnableMaxNumTokensTuningEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18DynamicBatchConfig27getEnableMaxNumTokensTuningEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18DynamicBatchConfig27getEnableMaxNumTokensTuningEv"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::getEnableMaxNumTokensTuningC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1ab0086c1cfad34346cd69749ae2ac753f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableMaxNumTokensTuning</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig27getEnableMaxNumTokensTuningEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig17getBatchSizeTableEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18DynamicBatchConfig17getBatchSizeTableEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18DynamicBatchConfig17getBatchSizeTableEv"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::getBatchSizeTableC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a874f07528531eff13ae54b28daf1b49b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBatchSizeTable</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig17getBatchSizeTableEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::kDefaultDynamicBatchMovingAverageWindow__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a384341c4c7bd1d935ec62852cb10290d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultDynamicBatchMovingAverageWindow</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">128</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The default window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::kDefaultBatchSizeTable__std::vector:std::pair:SizeType32.SizeType32::C"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a2552da34e844c7cbfcd9c4dd2fc38a7d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultBatchSizeTable</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The default value of batch size table. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22mEnableBatchSizeTuningE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig22mEnableBatchSizeTuningE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig22mEnableBatchSizeTuningE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::mEnableBatchSizeTuning__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a90b2eb7cff53639b2e35846ec27c771f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableBatchSizeTuning</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22mEnableBatchSizeTuningE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if the batch size should be tuned dynamically. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig25mEnableMaxNumTokensTuningE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig25mEnableMaxNumTokensTuningE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig25mEnableMaxNumTokensTuningE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::mEnableMaxNumTokensTuning__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a1a0e19a1020c36566d642eb0d5ec3a36"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableMaxNumTokensTuning</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig25mEnableMaxNumTokensTuningE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if the max num tokens should be tuned dynamically. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig32mDynamicBatchMovingAverageWindowE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig32mDynamicBatchMovingAverageWindowE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig32mDynamicBatchMovingAverageWindowE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::mDynamicBatchMovingAverageWindow__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a24b17e19c808eeabe3263296535d12b5"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDynamicBatchMovingAverageWindow</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig32mDynamicBatchMovingAverageWindowE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The window size for moving average of input and output length which is used to calculate dynamic batch size and max num tokens. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig15mBatchSizeTableE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18DynamicBatchConfig15mBatchSizeTableE"></span><span id="_CPPv2N12tensorrt_llm8executor18DynamicBatchConfig15mBatchSizeTableE"></span><span id="tensorrt_llm::executor::DynamicBatchConfig::mBatchSizeTable__std::vector:std::pair:SizeType32.SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DynamicBatchConfig_1a4050c896c0b2ac162d4745117ebb56a0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBatchSizeTable</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig15mBatchSizeTableE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>A vector of (batchSizeLimit, batchSize). When max capacity batch size is less than. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfigE"></span><span id="tensorrt_llm::executor::EagleConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig11EagleConfigENSt8optionalI12EagleChoicesEEbNSt8optionalIfEEbNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig11EagleConfigENSt8optionalI12EagleChoicesEEbNSt8optionalIfEEbNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig11EagleConfigENSt8optionalI12EagleChoicesEEbNSt8optionalIfEEbNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::EagleConfig::EagleConfig__std::optional:EagleChoices:.b.std::optional:float:.b.std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a1eed3b9134bfe3f263163591b647f01e"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="tensorrt_llm::executor::EagleChoices"><span class="n"><span class="pre">EagleChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">eagleChoices</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">greedySampling</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">posteriorThreshold</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useDynamicTree</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dynamicTreeMaxTopK</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig11EagleConfigENSt8optionalI12EagleChoicesEEbNSt8optionalIfEEbNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfigeqERK11EagleConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfigeqERK11EagleConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfigeqERK11EagleConfig"></span><span id="tensorrt_llm::executor::EagleConfig::eq-operator__EagleConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a911ae6f733aa6621787773db6875c888"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfigeqERK11EagleConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfig15getEagleChoicesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfig15getEagleChoicesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfig15getEagleChoicesEv"></span><span id="tensorrt_llm::executor::EagleConfig::getEagleChoicesC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1ada639d3d66b9a05b5a8804c39906c3f9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="tensorrt_llm::executor::EagleChoices"><span class="n"><span class="pre">EagleChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEagleChoices</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig15getEagleChoicesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfig21getPosteriorThresholdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfig21getPosteriorThresholdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfig21getPosteriorThresholdEv"></span><span id="tensorrt_llm::executor::EagleConfig::getPosteriorThresholdC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a39be08f82dd0b5387bc3ebd751b93084"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPosteriorThreshold</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig21getPosteriorThresholdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfig16isGreedySamplingEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfig16isGreedySamplingEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfig16isGreedySamplingEv"></span><span id="tensorrt_llm::executor::EagleConfig::isGreedySamplingC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a0bebe607080f2e6dbe6835df155dc4b3"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isGreedySampling</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig16isGreedySamplingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfig14useDynamicTreeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfig14useDynamicTreeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfig14useDynamicTreeEv"></span><span id="tensorrt_llm::executor::EagleConfig::useDynamicTreeC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a65f7dc9df51cd128154b941ad9ab780c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useDynamicTree</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig14useDynamicTreeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11EagleConfig21getDynamicTreeMaxTopKEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11EagleConfig21getDynamicTreeMaxTopKEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11EagleConfig21getDynamicTreeMaxTopKEv"></span><span id="tensorrt_llm::executor::EagleConfig::getDynamicTreeMaxTopKC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a1bbb46a0dc9dc6ee4358b6b42b476929"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDynamicTreeMaxTopK</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig21getDynamicTreeMaxTopKEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig19checkPosteriorValueERKNSt8optionalIfEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig19checkPosteriorValueERKNSt8optionalIfEE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig19checkPosteriorValueERKNSt8optionalIfEE"></span><span id="tensorrt_llm::executor::EagleConfig::checkPosteriorValue__std::optional:float:CR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1acc18740b06d0c59ba7481a4a8e549f3e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkPosteriorValue</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">value</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19checkPosteriorValueERKNSt8optionalIfEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig13mEagleChoicesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig13mEagleChoicesE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig13mEagleChoicesE"></span><span id="tensorrt_llm::executor::EagleConfig::mEagleChoices__std::optional:EagleChoices:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1afb9fa382408155bdafb7a611d37b7931"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="tensorrt_llm::executor::EagleChoices"><span class="n"><span class="pre">EagleChoices</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEagleChoices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig13mEagleChoicesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>choices forming tree for EAGLE-1. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig15mGreedySamplingE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig15mGreedySamplingE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig15mGreedySamplingE"></span><span id="tensorrt_llm::executor::EagleConfig::mGreedySampling__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a36836b50046667b4ca54bb120aaa1998"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGreedySampling</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig15mGreedySamplingE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Flag to use greedy or typical acceptance. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig19mPosteriorThresholdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig19mPosteriorThresholdE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig19mPosteriorThresholdE"></span><span id="tensorrt_llm::executor::EagleConfig::mPosteriorThreshold__std::optional:float:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1ad26ec15a330911822577a7a483b0a1d7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPosteriorThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19mPosteriorThresholdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Minimum token probability of the typical acceptance. Corresponds to epsilon in <a class="reference external" href="https://arxiv.org/pdf/2401.10774">https://arxiv.org/pdf/2401.10774</a>. Default is 0.09f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig15mUseDynamicTreeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig15mUseDynamicTreeE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig15mUseDynamicTreeE"></span><span id="tensorrt_llm::executor::EagleConfig::mUseDynamicTree__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1afa25213f577836b4a6e4907dd98d1bf4"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseDynamicTree</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig15mUseDynamicTreeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Flag to use Eagle-2. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11EagleConfig19mDynamicTreeMaxTopKE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11EagleConfig19mDynamicTreeMaxTopKE"></span><span id="_CPPv2N12tensorrt_llm8executor11EagleConfig19mDynamicTreeMaxTopKE"></span><span id="tensorrt_llm::executor::EagleConfig::mDynamicTreeMaxTopK__std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1EagleConfig_1a57f41c0f1eaace45df3b5ed27286896d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDynamicTreeMaxTopK</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19mDynamicTreeMaxTopKE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of draft tokens expand for each node in Eagle-2. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ExecutorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ExecutorE"></span><span id="_CPPv2N12tensorrt_llm8executor8ExecutorE"></span><span id="tensorrt_llm::executor::Executor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>The executor is responsible for receiving new requests and sending responses, and running the inference. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Executor::Executor__std::filesystem::pathCR.ModelType.ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a74e112aaa1be8f0b62f8d96f59021e1a"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">modelPath</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="tensorrt_llm::executor::ModelType"><span class="n"><span class="pre">ModelType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd><dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>modelPath</strong> – Path to the folder that defines the model to run </p></li>
|
||
<li><p><strong>modelType</strong> – The type of model </p></li>
|
||
<li><p><strong>executorConfig</strong> – The configuration for the executor </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Executor::Executor__std::filesystem::pathCR.std::filesystem::pathCR.ModelType.ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1af2fb4913e74a0cc0b3a85ef58bd1f662"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">encoderModelPath</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decoderModelPath</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="tensorrt_llm::executor::ModelType"><span class="n"><span class="pre">ModelType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfigRKNSt8optionalINSt3mapINSt6stringE6TensorEEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfigRKNSt8optionalINSt3mapINSt6stringE6TensorEEEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfigRKNSt8optionalINSt3mapINSt6stringE6TensorEEEE"></span><span id="tensorrt_llm::executor::Executor::Executor__BufferViewCR.ssCR.ModelType.ExecutorConfigCR.std::optional:std::map:ss.Tensor::CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1ab16c36317e9f3389f6d6ca0490e892d9"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BufferViewE" title="tensorrt_llm::executor::BufferView"><span class="n"><span class="pre">BufferView</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">engineBuffer</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">jsonConfigStr</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="tensorrt_llm::executor::ModelType"><span class="n"><span class="pre">ModelType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">managedWeights</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfigRKNSt8optionalINSt3mapINSt6stringE6TensorEEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Executor::Executor__BufferViewCR.ssCR.BufferViewCR.ssCR.ModelType.ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1af79e2bea60216502ba77f33508327300"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BufferViewE" title="tensorrt_llm::executor::BufferView"><span class="n"><span class="pre">BufferView</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">encoderEngineBuffer</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">encoderJsonConfigStr</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BufferViewE" title="tensorrt_llm::executor::BufferView"><span class="n"><span class="pre">BufferView</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decoderEngineBuffer</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decoderJsonConfigStr</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE" title="tensorrt_llm::executor::ModelType"><span class="n"><span class="pre">ModelType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEERK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEERK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEERK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Executor::Executor__std::shared_ptr:Model:.ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a9a98e9c79f95e75c2a16f900986082ce"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Model</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">model</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEERK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEENSt10shared_ptrI5ModelEERK14ExecutorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEENSt10shared_ptrI5ModelEERK14ExecutorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEENSt10shared_ptrI5ModelEERK14ExecutorConfig"></span><span id="tensorrt_llm::executor::Executor::Executor__std::shared_ptr:Model:.std::shared_ptr:Model:.ExecutorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1aedb3a745955f57fa268a326e49c0f741"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Model</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">encoderModel</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Model</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">decoderModel</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="tensorrt_llm::executor::ExecutorConfig"><span class="n"><span class="pre">ExecutorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEENSt10shared_ptrI5ModelEERK14ExecutorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ExecutorD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ExecutorD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8ExecutorD0Ev"></span><span id="tensorrt_llm::executor::Executor::~Executor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a7c5a64a1cc125f8b6a17f05c85530b57"></span><span class="sig-name descname"><span class="n"><span class="pre">~Executor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ExecutorD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK8Executor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERK8Executor"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERK8Executor"></span><span id="tensorrt_llm::executor::Executor::Executor__ExecutorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1af5a2778fb124f2e594c2f6e30a6e8322"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK8Executor" title="tensorrt_llm::executor::Executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executor</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK8Executor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ExecutoraSERK8Executor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ExecutoraSERK8Executor"></span><span id="_CPPv2N12tensorrt_llm8executor8ExecutoraSERK8Executor"></span><span id="tensorrt_llm::executor::Executor::assign-operator__ExecutorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a375b8e9c2ec8f50865044eea22f949fb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE" title="tensorrt_llm::executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE" title="tensorrt_llm::executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">executor</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ExecutoraSERK8Executor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERR8Executor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8ExecutorERR8Executor"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8ExecutorERR8Executor"></span><span id="tensorrt_llm::executor::Executor::Executor__ExecutorRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a5d8813384914105ab93bc5afc6ef633e"></span><span class="sig-name descname"><span class="n"><span class="pre">Executor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERR8Executor" title="tensorrt_llm::executor::Executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERR8Executor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ExecutoraSERR8Executor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ExecutoraSERR8Executor"></span><span id="_CPPv2N12tensorrt_llm8executor8ExecutoraSERR8Executor"></span><span id="tensorrt_llm::executor::Executor::assign-operator__ExecutorRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1ab35c6ac71a9d0846232caebcd39db3ad"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE" title="tensorrt_llm::executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE" title="tensorrt_llm::executor::Executor"><span class="n"><span class="pre">Executor</span></span></a><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ExecutoraSERR8Executor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor14enqueueRequestERK7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor14enqueueRequestERK7Request"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor14enqueueRequestERK7Request"></span><span id="tensorrt_llm::executor::Executor::enqueueRequest__RequestCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1af760463b7371a09773207283f9f0fbb6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enqueueRequest</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">request</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor14enqueueRequestERK7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enqueue a new request. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>request</strong> – The LLM request which contains input tokens and request parameters </p>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A unique id that identifies the request </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor15enqueueRequestsERKNSt6vectorI7RequestEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor15enqueueRequestsERKNSt6vectorI7RequestEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor15enqueueRequestsERKNSt6vectorI7RequestEE"></span><span id="tensorrt_llm::executor::Executor::enqueueRequests__std::vector:Request:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1af9acbedea80c9d77d8c77d9f66a74adb"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enqueueRequests</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requests</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor15enqueueRequestsERKNSt6vectorI7RequestEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enqueue a batch of request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::Executor::awaitResponses__std::optional:std::chrono::milliseconds:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a163139b9705933beade11af62468f127"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">awaitResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">timeout</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Await for ready responses. </p>
|
||
<p><div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="n">This</span> <span class="n">overload</span> <span class="n">awaits</span> <span class="k">for</span> <span class="nb">any</span> <span class="n">ready</span> <span class="n">responses</span><span class="o">.</span> <span class="n">In</span> <span class="n">particular</span><span class="p">,</span> <span class="k">if</span> <span class="n">several</span> <span class="n">requests</span>
|
||
<span class="n">have</span> <span class="n">been</span> <span class="n">enqueued</span><span class="p">,</span> <span class="n">this</span> <span class="n">method</span> <span class="n">will</span> <span class="n">provide</span> <span class="nb">any</span> <span class="n">ready</span> <span class="n">responses</span> <span class="n">without</span> <span class="n">order</span> <span class="n">guarantees</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>timeout</strong> – The maximum time to wait for new responses </p>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of responses </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERK6IdTypeRKNSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor14awaitResponsesERK6IdTypeRKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor14awaitResponsesERK6IdTypeRKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::Executor::awaitResponses__IdTypeCR.std::optional:std::chrono::milliseconds:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1ab08f8c1d216d8e9f9c8a214d72d69f65"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">awaitResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">timeout</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERK6IdTypeRKNSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Await for ready responses. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>id</strong> – A request id </p></li>
|
||
<li><p><strong>timeout</strong> – The maximum time to wait for new responses </p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of responses </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt6vectorI6IdTypeEERKNSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt6vectorI6IdTypeEERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt6vectorI6IdTypeEERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::Executor::awaitResponses__std::vector:IdType:CR.std::optional:std::chrono::milliseconds:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a8954fd92f602eab2467de3f1cd419661"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">awaitResponses</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestIds</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">timeout</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt6vectorI6IdTypeEERKNSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Await for multiple ready responses. </p>
|
||
<p><div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="n">A</span> <span class="n">multiple</span> <span class="n">ID</span> <span class="n">request</span> <span class="n">behaves</span> <span class="k">as</span> <span class="k">if</span> <span class="n">awaitResponses</span><span class="p">(</span><span class="n">IdType</span><span class="p">,</span> <span class="n">timeout</span><span class="p">)</span>
|
||
<span class="n">were</span> <span class="n">invoked</span> <span class="n">on</span> <span class="nb">all</span> <span class="n">IDs</span><span class="o">.</span> <span class="n">The</span> <span class="n">returned</span> <span class="n">vector</span> <span class="n">contains</span>
|
||
<span class="n">a</span> <span class="n">vector</span> <span class="n">of</span> <span class="n">responses</span> <span class="n">per</span> <span class="n">ID</span> <span class="ow">in</span> <span class="n">the</span> <span class="n">same</span> <span class="n">order</span> <span class="n">specified</span> <span class="n">by</span> <span class="n">the</span> <span class="n">requestIds</span><span class="o">.</span>
|
||
<span class="n">The</span> <span class="n">same</span> <span class="n">behaviour</span> <span class="k">as</span> <span class="n">awaitResponses</span><span class="p">(</span><span class="n">IdType</span><span class="p">,</span> <span class="n">timeout</span><span class="p">)</span> <span class="n">applies</span><span class="p">:</span>
|
||
<span class="o">*</span> <span class="n">Responses</span> <span class="n">may</span> <span class="n">be</span> <span class="n">empty</span><span class="o">.</span>
|
||
<span class="o">*</span> <span class="n">If</span> <span class="nb">all</span> <span class="n">responses</span> <span class="n">have</span> <span class="n">already</span> <span class="n">been</span> <span class="n">given</span> <span class="k">for</span> <span class="n">one</span> <span class="n">of</span> <span class="n">the</span> <span class="n">requestIds</span><span class="p">,</span>
|
||
<span class="n">then</span> <span class="n">this</span> <span class="n">method</span> <span class="n">will</span> <span class="n">hang</span> <span class="n">unless</span> <span class="n">a</span> <span class="n">timeout</span> <span class="ow">is</span> <span class="n">specified</span><span class="o">.</span>
|
||
</pre></div>
|
||
</div>
|
||
</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>requestIds</strong> – Ids requested </p></li>
|
||
<li><p><strong>timeout</strong> – The maximum time to wait for new responses </p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>A vector of vector of responses </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Executor20getNumResponsesReadyERKNSt8optionalI6IdTypeEE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Executor20getNumResponsesReadyERKNSt8optionalI6IdTypeEE"></span><span id="_CPPv2NK12tensorrt_llm8executor8Executor20getNumResponsesReadyERKNSt8optionalI6IdTypeEE"></span><span id="tensorrt_llm::executor::Executor::getNumResponsesReady__std::optional:IdType:CRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a8870564de9e1b838e3a24e770de65e97"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumResponsesReady</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Executor20getNumResponsesReadyERKNSt8optionalI6IdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the number of ready responses. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>requestId</strong> – An optional request id </p>
|
||
</dd>
|
||
<dt class="field-even">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p>The number of ready responses </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor13cancelRequestE6IdType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor13cancelRequestE6IdType"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor13cancelRequestE6IdType"></span><span id="tensorrt_llm::executor::Executor::cancelRequest__IdType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1ad157e5c600e0bcd1fb16695d573ec279"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cancelRequest</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">requestId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor13cancelRequestE6IdType" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Cancel the request with provided request id. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>id</strong> – The request id for which to cancel the response </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor8shutdownEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor8shutdownEv"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor8shutdownEv"></span><span id="tensorrt_llm::executor::Executor::shutdown"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1abee0bdae43e18a83ada60f4e63eb72d5"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shutdown</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor8shutdownEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Signals the server to shutdown. </p>
|
||
<p>This call is blocking. Only returns when all requests have terminated or timeout has been reached </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor23getLatestIterationStatsEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor23getLatestIterationStatsEv"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor23getLatestIterationStatsEv"></span><span id="tensorrt_llm::executor::Executor::getLatestIterationStats"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a86d2af2924046fdff9fb70a9e86159bd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLatestIterationStats</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor23getLatestIterationStatsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the per-iterations statistics computed since last call to getLatestIterationStats. Contains at most iterStatsMaxIterations iterations. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>Iteration stats </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor21getLatestRequestStatsEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor21getLatestRequestStatsEv"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor21getLatestRequestStatsEv"></span><span id="tensorrt_llm::executor::Executor::getLatestRequestStats"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1ab75ad0e4223d528aa87fdd18d1caeec0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLatestRequestStats</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor21getLatestRequestStatsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the request stats of each iteration computed since last call to getLatestRequestStats. Contains at most requestStatsMaxIterations iterations. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1Request"><span class="std std-ref">Request</span></a> stats grouped by iterations </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor21getLatestDebugTensorsEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor21getLatestDebugTensorsEv"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor21getLatestDebugTensorsEv"></span><span id="tensorrt_llm::executor::Executor::getLatestDebugTensors"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a65398075a94e7065f0c5a0d7ae4e4538"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIterationE" title="tensorrt_llm::executor::DebugTensorsPerIteration"><span class="n"><span class="pre">DebugTensorsPerIteration</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLatestDebugTensors</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor21getLatestDebugTensorsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the debug tensors of each iteration computed since last call to getLatestDebugTensors. Contains at most debugTensorsMaxIterations iterations. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Returns<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1Request"><span class="std std-ref">Request</span></a> debug tensors grouped by iterations </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Executor18canEnqueueRequestsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Executor18canEnqueueRequestsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Executor18canEnqueueRequestsEv"></span><span id="tensorrt_llm::executor::Executor::canEnqueueRequestsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a385dcd6e4c171cc8dcc6b7e327a8524b"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">canEnqueueRequests</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Executor18canEnqueueRequestsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if the current process is allowed to enqueueRequests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Executor13isParticipantEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Executor13isParticipantEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Executor13isParticipantEv"></span><span id="tensorrt_llm::executor::Executor::isParticipantC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a2b6ddd22f24a00cf5eeaafce158aaf64"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isParticipant</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Executor13isParticipantEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if the current process participates in this executor instance. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Executor22getKVCacheEventManagerEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Executor22getKVCacheEventManagerEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Executor22getKVCacheEventManagerEv"></span><span id="tensorrt_llm::executor::Executor::getKVCacheEventManagerC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a4c21f8a3611d700e1757ac2fb4348836"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManagerE" title="tensorrt_llm::executor::KVCacheEventManager"><span class="n"><span class="pre">KVCacheEventManager</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKVCacheEventManager</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Executor22getKVCacheEventManagerEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Executor5mImplE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Executor5mImplE"></span><span id="_CPPv2N12tensorrt_llm8executor8Executor5mImplE"></span><span id="tensorrt_llm::executor::Executor::mImpl__std::unique_ptr:Impl:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Executor_1a4b7affd37a154513db81d23acb941801"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Impl</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mImpl</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Executor5mImplE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExecutorConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for the model executor. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig14ExecutorConfigE10SizeType3215SchedulerConfig13KvCacheConfigbb10SizeType3210SizeType3212BatchingTypeNSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI14ParallelConfigEERKNSt8optionalI15PeftCacheConfigEENSt8optionalI25LogitsPostProcessorConfigEENSt8optionalI14DecodingConfigEEbfNSt8optionalI10SizeType32EERK29ExtendedRuntimePerfKnobConfigNSt8optionalI11DebugConfigEE10SizeType328uint64_tNSt8optionalI25SpeculativeDecodingConfigEENSt8optionalI20GuidedDecodingConfigEENSt8optionalINSt6vectorI21AdditionalModelOutputEEEENSt8optionalI22CacheTransceiverConfigEEbbbb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig14ExecutorConfigE10SizeType3215SchedulerConfig13KvCacheConfigbb10SizeType3210SizeType3212BatchingTypeNSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI14ParallelConfigEERKNSt8optionalI15PeftCacheConfigEENSt8optionalI25LogitsPostProcessorConfigEENSt8optionalI14DecodingConfigEEbfNSt8optionalI10SizeType32EERK29ExtendedRuntimePerfKnobConfigNSt8optionalI11DebugConfigEE10SizeType328uint64_tNSt8optionalI25SpeculativeDecodingConfigEENSt8optionalI20GuidedDecodingConfigEENSt8optionalINSt6vectorI21AdditionalModelOutputEEEENSt8optionalI22CacheTransceiverConfigEEbbbb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig14ExecutorConfigE10SizeType3215SchedulerConfig13KvCacheConfigbb10SizeType3210SizeType3212BatchingTypeNSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI14ParallelConfigEERKNSt8optionalI15PeftCacheConfigEENSt8optionalI25LogitsPostProcessorConfigEENSt8optionalI14DecodingConfigEEbfNSt8optionalI10SizeType32EERK29ExtendedRuntimePerfKnobConfigNSt8optionalI11DebugConfigEE10SizeType328uint64_tNSt8optionalI25SpeculativeDecodingConfigEENSt8optionalI20GuidedDecodingConfigEENSt8optionalINSt6vectorI21AdditionalModelOutputEEEENSt8optionalI22CacheTransceiverConfigEEbbbb"></span><span id="tensorrt_llm::executor::ExecutorConfig::ExecutorConfig__SizeType32.SchedulerConfig.KvCacheConfig.b.b.SizeType32.SizeType32.BatchingType.std::optional:SizeType32:.std::optional:SizeType32:.std::optional:ParallelConfig:.std::optional:PeftCacheConfig:CR.std::optional:LogitsPostProcessorConfig:.std::optional:DecodingConfig:.b.float.std::optional:SizeType32:.ExtendedRuntimePerfKnobConfigCR.std::optional:DebugConfig:.SizeType32.uint64_t.std::optional:SpeculativeDecodingConfig:.std::optional:GuidedDecodingConfig:.std::optional:std::vector:AdditionalModelOutput::.std::optional:CacheTransceiverConfig:.b.b.b.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a70e4676c00ae2f3506c7c14904174710"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExecutorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">schedulerConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">kvCacheConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableChunkedContext</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">normalizeLogProbs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">iterStatsMaxIterations</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE" title="tensorrt_llm::executor::ExecutorConfig::kDefaultIterStatsMaxIterations"><span class="n"><span class="pre">kDefaultIterStatsMaxIterations</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">requestStatsMaxIterations</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE" title="tensorrt_llm::executor::ExecutorConfig::kDefaultRequestStatsMaxIterations"><span class="n"><span class="pre">kDefaultRequestStatsMaxIterations</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="tensorrt_llm::executor::BatchingType"><span class="n"><span class="pre">BatchingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchingType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="tensorrt_llm::executor::BatchingType"><span class="n"><span class="pre">BatchingType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingType9kINFLIGHTE" title="tensorrt_llm::executor::BatchingType::kINFLIGHT"><span class="n"><span class="pre">kINFLIGHT</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">parallelConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">peftCacheConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE" title="tensorrt_llm::executor::LogitsPostProcessorConfig"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">logitsPostProcessorConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">decodingConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useGpuDirectStorage</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">gpuWeightsPercent</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxQueueSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">extendedRuntimePerfKnobConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">debugConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">recvPollPeriodMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxSeqIdleMicroseconds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE" title="tensorrt_llm::executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds"><span class="n"><span class="pre">kDefaultMaxSeqIdleMicroseconds</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">specDecConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">guidedDecodingConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">additionalModelOutputs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">cacheTransceiverConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">gatherGenerationLogits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">promptTableOffloading</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableTrtOverlap</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">failFastOnAttentionWindowTooLarge</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14ExecutorConfigE10SizeType3215SchedulerConfig13KvCacheConfigbb10SizeType3210SizeType3212BatchingTypeNSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI14ParallelConfigEERKNSt8optionalI15PeftCacheConfigEENSt8optionalI25LogitsPostProcessorConfigEENSt8optionalI14DecodingConfigEEbfNSt8optionalI10SizeType32EERK29ExtendedRuntimePerfKnobConfigNSt8optionalI11DebugConfigEE10SizeType328uint64_tNSt8optionalI25SpeculativeDecodingConfigEENSt8optionalI20GuidedDecodingConfigEENSt8optionalINSt6vectorI21AdditionalModelOutputEEEENSt8optionalI22CacheTransceiverConfigEEbbbb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBeamWidthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig15getMaxBeamWidthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig15getMaxBeamWidthEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getMaxBeamWidthC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af28528ea0a4f5d8252d5cf4aa809cab9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBeamWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBeamWidthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getSchedulerConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig18getSchedulerConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig18getSchedulerConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getSchedulerConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ad04adf7773f5dc011df7811ed71a2065"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSchedulerConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getSchedulerConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getKvCacheConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig16getKvCacheConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig16getKvCacheConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getKvCacheConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a81554cd8e28a7c975dcb88e0bc678829"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKvCacheConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getKvCacheConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig21getSchedulerConfigRefEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig21getSchedulerConfigRefEv"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig21getSchedulerConfigRefEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getSchedulerConfigRef"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ac21502045320dd9d10638451f7a75488"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSchedulerConfigRef</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21getSchedulerConfigRefEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig19getKvCacheConfigRefEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig19getKvCacheConfigRefEv"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig19getKvCacheConfigRefEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getKvCacheConfigRef"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a71c8f73c26ef6e876eefdc78128c4b5d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getKvCacheConfigRef</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19getKvCacheConfigRefEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getEnableChunkedContextEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig23getEnableChunkedContextEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig23getEnableChunkedContextEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getEnableChunkedContextC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1acebbdef0320ea601a14d69ee84b0d8e8"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableChunkedContext</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getEnableChunkedContextEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getNormalizeLogProbsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig20getNormalizeLogProbsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig20getNormalizeLogProbsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getNormalizeLogProbsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af9638fec77c9ad7a4a9230c6b8ccd3e2"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNormalizeLogProbs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getNormalizeLogProbsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getIterStatsMaxIterationsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig25getIterStatsMaxIterationsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig25getIterStatsMaxIterationsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getIterStatsMaxIterationsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ad68d1ec95040a53535b2df4bbc5e67ad"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIterStatsMaxIterations</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getIterStatsMaxIterationsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getRequestStatsMaxIterationsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig28getRequestStatsMaxIterationsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig28getRequestStatsMaxIterationsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getRequestStatsMaxIterationsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a80fbe7fc988029b4e5605bc79d8983e5"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRequestStatsMaxIterations</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getRequestStatsMaxIterationsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getBatchingTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig15getBatchingTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig15getBatchingTypeEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getBatchingTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ac90fe84eae462d1eae8d2ad84c4c8bee"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="tensorrt_llm::executor::BatchingType"><span class="n"><span class="pre">BatchingType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBatchingType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getBatchingTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBatchSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig15getMaxBatchSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig15getMaxBatchSizeEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getMaxBatchSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a602a24b9c115d92f33b59045a9f9dbd7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBatchSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBatchSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxNumTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig15getMaxNumTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig15getMaxNumTokensEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getMaxNumTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a201dbdac3a1cdd40d766f966f42905c4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxNumTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxNumTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getParallelConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig17getParallelConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig17getParallelConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getParallelConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a301d8dae1d5715022c95eda417db6c6e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getParallelConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getParallelConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getPeftCacheConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig18getPeftCacheConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig18getPeftCacheConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getPeftCacheConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a6e6e228a04ba55d677e1903c24ca9cb0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPeftCacheConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getPeftCacheConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getLogitsPostProcessorConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig28getLogitsPostProcessorConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig28getLogitsPostProcessorConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getLogitsPostProcessorConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a442a13ce5bc822eda741b7677d0b6709"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE" title="tensorrt_llm::executor::LogitsPostProcessorConfig"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogitsPostProcessorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getLogitsPostProcessorConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getDecodingConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig17getDecodingConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig17getDecodingConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getDecodingConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a3b3bc02cbbf6c525d2643c0206409785"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDecodingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getDecodingConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig22getUseGpuDirectStorageEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig22getUseGpuDirectStorageEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig22getUseGpuDirectStorageEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getUseGpuDirectStorageC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af6de84a614885ecd93071dad48643e2a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getUseGpuDirectStorage</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig22getUseGpuDirectStorageEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getGpuWeightsPercentEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig20getGpuWeightsPercentEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig20getGpuWeightsPercentEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getGpuWeightsPercentC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a45340052913aba3d6e3acdf12b07d4fd"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpuWeightsPercent</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getGpuWeightsPercentEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxQueueSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig15getMaxQueueSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig15getMaxQueueSizeEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getMaxQueueSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a7a79bd90a5dee83095bd3c6596d4b463"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxQueueSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxQueueSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig32getExtendedRuntimePerfKnobConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig32getExtendedRuntimePerfKnobConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig32getExtendedRuntimePerfKnobConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getExtendedRuntimePerfKnobConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a7713e188a26de4fb490ad99a4eb385e6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getExtendedRuntimePerfKnobConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig32getExtendedRuntimePerfKnobConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig14getDebugConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig14getDebugConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig14getDebugConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getDebugConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a412b3fcaeb31bdeddfe73a1be474d450"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDebugConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig14getDebugConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getRecvPollPeriodMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig19getRecvPollPeriodMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig19getRecvPollPeriodMsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getRecvPollPeriodMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a8f75db7250ff15d1255a56915234cb35"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRecvPollPeriodMs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getRecvPollPeriodMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getMaxSeqIdleMicrosecondsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig25getMaxSeqIdleMicrosecondsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig25getMaxSeqIdleMicrosecondsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getMaxSeqIdleMicrosecondsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a7a9d21d7e4b3aa5e0ae43011f520d668"></span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxSeqIdleMicroseconds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getMaxSeqIdleMicrosecondsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getSpecDecConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig16getSpecDecConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig16getSpecDecConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getSpecDecConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a704988c0c750a3321f0c89a5477914dd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpecDecConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getSpecDecConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getGuidedDecodingConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig23getGuidedDecodingConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig23getGuidedDecodingConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getGuidedDecodingConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4fe6cf353990885c3eb41bef8d6a7e80"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGuidedDecodingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getGuidedDecodingConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getAdditionalModelOutputsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig25getAdditionalModelOutputsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig25getAdditionalModelOutputsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getAdditionalModelOutputsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a315fe34df6a53bdb1eec0ae9bc1c607e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAdditionalModelOutputs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getAdditionalModelOutputsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getGatherGenerationLogitsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig25getGatherGenerationLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig25getGatherGenerationLogitsEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getGatherGenerationLogitsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a49a1b0247c3f1c5a457d801c5d838fdc"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGatherGenerationLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getGatherGenerationLogitsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig24getPromptTableOffloadingEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig24getPromptTableOffloadingEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig24getPromptTableOffloadingEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getPromptTableOffloadingC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4a127c06c660bb733e0df224c1c8e91b"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPromptTableOffloading</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig24getPromptTableOffloadingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getCacheTransceiverConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig25getCacheTransceiverConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig25getCacheTransceiverConfigEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getCacheTransceiverConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a0a1869d9ac41e6bcadca11c7ca621e2d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCacheTransceiverConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getCacheTransceiverConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getEnableTrtOverlapEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig19getEnableTrtOverlapEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig19getEnableTrtOverlapEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getEnableTrtOverlapC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9079fa3a603e9dd35c1bd0c38f510c31"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableTrtOverlap</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getEnableTrtOverlapEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ExecutorConfig36getFailFastOnAttentionWindowTooLargeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ExecutorConfig36getFailFastOnAttentionWindowTooLargeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ExecutorConfig36getFailFastOnAttentionWindowTooLargeEv"></span><span id="tensorrt_llm::executor::ExecutorConfig::getFailFastOnAttentionWindowTooLargeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a650989eb45ce44438ee90a7e4593ab50"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFailFastOnAttentionWindowTooLarge</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig36getFailFastOnAttentionWindowTooLargeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBeamWidthE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15setMaxBeamWidthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15setMaxBeamWidthE10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setMaxBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4a9426ca428d09d9514e85a0c2b612f5"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxBeamWidth</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBeamWidthE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBatchSizeE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15setMaxBatchSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15setMaxBatchSizeE10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setMaxBatchSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1acbccae1669e096e7214a532e8ca415dc"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxBatchSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBatchSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxNumTokensE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15setMaxNumTokensE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15setMaxNumTokensE10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setMaxNumTokens__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af273f0c3f408e7b54354b00a9d69ca3a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxNumTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumTokens</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxNumTokensE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setSchedulerConfigERK15SchedulerConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig18setSchedulerConfigERK15SchedulerConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig18setSchedulerConfigERK15SchedulerConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setSchedulerConfig__SchedulerConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a3d0b81d8422cbb579ba37d425c600004"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSchedulerConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">schedulerConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setSchedulerConfigERK15SchedulerConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setKvCacheConfigERK13KvCacheConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig16setKvCacheConfigERK13KvCacheConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig16setKvCacheConfigERK13KvCacheConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setKvCacheConfig__KvCacheConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1abdfb27a2cc58da0a28ce5c2486ea688f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setKvCacheConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setKvCacheConfigERK13KvCacheConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setEnableChunkedContextEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23setEnableChunkedContextEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23setEnableChunkedContextEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setEnableChunkedContext__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a6426dbe445f65be08fae30ddc03135f4"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEnableChunkedContext</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableChunkedContext</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setEnableChunkedContextEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setNormalizeLogProbsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig20setNormalizeLogProbsEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig20setNormalizeLogProbsEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setNormalizeLogProbs__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aa40eff9c23c0abd6d4be7977bb80d80a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNormalizeLogProbs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">normalizeLogProbs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setNormalizeLogProbsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setIterStatsMaxIterationsE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig25setIterStatsMaxIterationsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig25setIterStatsMaxIterationsE10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setIterStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1acab1aa5f729a9d145d23c0a0c8c2e526"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setIterStatsMaxIterations</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">iterStatsMaxIterations</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setIterStatsMaxIterationsE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setRequestStatsMaxIterationsE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig28setRequestStatsMaxIterationsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig28setRequestStatsMaxIterationsE10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setRequestStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1acbb1bfe2c8a77c80d3ec6e55ff00196a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRequestStatsMaxIterations</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">requestStatsMaxIterations</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setRequestStatsMaxIterationsE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setBatchingTypeE12BatchingType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15setBatchingTypeE12BatchingType"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15setBatchingTypeE12BatchingType"></span><span id="tensorrt_llm::executor::ExecutorConfig::setBatchingType__BatchingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a7171a21854120d745c185d6d2ecd9673"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBatchingType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="tensorrt_llm::executor::BatchingType"><span class="n"><span class="pre">BatchingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchingType</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setBatchingTypeE12BatchingType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setParallelConfigERK14ParallelConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig17setParallelConfigERK14ParallelConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig17setParallelConfigERK14ParallelConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setParallelConfig__ParallelConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a36697066f6b8a8e7ea8d031ce1bfdaca"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setParallelConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">parallelConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setParallelConfigERK14ParallelConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setPeftCacheConfigERK15PeftCacheConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig18setPeftCacheConfigERK15PeftCacheConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig18setPeftCacheConfigERK15PeftCacheConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setPeftCacheConfig__PeftCacheConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af5453098e1af9097cc47acb28d1df39a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPeftCacheConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">peftCacheConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setPeftCacheConfigERK15PeftCacheConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setLogitsPostProcessorConfigERK25LogitsPostProcessorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig28setLogitsPostProcessorConfigERK25LogitsPostProcessorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig28setLogitsPostProcessorConfigERK25LogitsPostProcessorConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setLogitsPostProcessorConfig__LogitsPostProcessorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aabc5cd613e383abd181d7b8e6347a1d2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLogitsPostProcessorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE" title="tensorrt_llm::executor::LogitsPostProcessorConfig"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">logitsPostProcessorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setLogitsPostProcessorConfigERK25LogitsPostProcessorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setDecodingConfigERK14DecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig17setDecodingConfigERK14DecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig17setDecodingConfigERK14DecodingConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setDecodingConfig__DecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a7d6173ec1a87d222ff1674d047b3d69b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDecodingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">decodingConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setDecodingConfigERK14DecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig22setUseGpuDirectStorageERKb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig22setUseGpuDirectStorageERKb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig22setUseGpuDirectStorageERKb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setUseGpuDirectStorage__bCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a40239ffe54809a5afd16d71ae17c2022"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseGpuDirectStorage</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">useGpuDirectStorage</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig22setUseGpuDirectStorageERKb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setGpuWeightsPercentERKf">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig20setGpuWeightsPercentERKf"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig20setGpuWeightsPercentERKf"></span><span id="tensorrt_llm::executor::ExecutorConfig::setGpuWeightsPercent__floatCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a555f875ac320973a0e0bc94e0e9611b1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setGpuWeightsPercent</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">gpuWeightsPercent</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setGpuWeightsPercentERKf" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxQueueSizeERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15setMaxQueueSizeERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15setMaxQueueSizeERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::ExecutorConfig::setMaxQueueSize__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ad3ae79600c15c02fd14ddb5642ed0455"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxQueueSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">maxQueueSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxQueueSizeERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig32setExtendedRuntimePerfKnobConfigERK29ExtendedRuntimePerfKnobConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig32setExtendedRuntimePerfKnobConfigERK29ExtendedRuntimePerfKnobConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig32setExtendedRuntimePerfKnobConfigERK29ExtendedRuntimePerfKnobConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setExtendedRuntimePerfKnobConfig__ExtendedRuntimePerfKnobConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ae22a98cd8cc64b1caaab9463f2e3ae44"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setExtendedRuntimePerfKnobConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">extendedRuntimePerfKnobConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig32setExtendedRuntimePerfKnobConfigERK29ExtendedRuntimePerfKnobConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig14setDebugConfigERK11DebugConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig14setDebugConfigERK11DebugConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig14setDebugConfigERK11DebugConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setDebugConfig__DebugConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a2644f7d8f8156fab18febe21da42c6a4"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDebugConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">debugConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14setDebugConfigERK11DebugConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setRecvPollPeriodMsERK10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig19setRecvPollPeriodMsERK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig19setRecvPollPeriodMsERK10SizeType32"></span><span id="tensorrt_llm::executor::ExecutorConfig::setRecvPollPeriodMs__SizeType32CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ae716a2e12980f78e8d001e629a84663f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRecvPollPeriodMs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">recvPollPeriodMs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setRecvPollPeriodMsERK10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setMaxSeqIdleMicrosecondsE8uint64_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig25setMaxSeqIdleMicrosecondsE8uint64_t"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig25setMaxSeqIdleMicrosecondsE8uint64_t"></span><span id="tensorrt_llm::executor::ExecutorConfig::setMaxSeqIdleMicroseconds__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9dbc05aefd87a5f26c8ad0f8c7a3cffe"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxSeqIdleMicroseconds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxSeqIdleMicroseconds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setMaxSeqIdleMicrosecondsE8uint64_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setSpecDecConfigERK25SpeculativeDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig16setSpecDecConfigERK25SpeculativeDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig16setSpecDecConfigERK25SpeculativeDecodingConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setSpecDecConfig__SpeculativeDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1af7a9647b5d113457510b08ab62f6ab92"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSpecDecConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">specDecConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setSpecDecConfigERK25SpeculativeDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setGuidedDecodingConfigERK20GuidedDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23setGuidedDecodingConfigERK20GuidedDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23setGuidedDecodingConfigERK20GuidedDecodingConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setGuidedDecodingConfig__GuidedDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ad65dc27bdc3214e55abc302239535419"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setGuidedDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setGuidedDecodingConfigERK20GuidedDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setAdditionalModelOutputsERKNSt6vectorI21AdditionalModelOutputEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig25setAdditionalModelOutputsERKNSt6vectorI21AdditionalModelOutputEE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig25setAdditionalModelOutputsERKNSt6vectorI21AdditionalModelOutputEE"></span><span id="tensorrt_llm::executor::ExecutorConfig::setAdditionalModelOutputs__std::vector:AdditionalModelOutput:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a5e2c7d95cc02f6befb559d7088858d27"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setAdditionalModelOutputs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">additionalModelOutputs</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setAdditionalModelOutputsERKNSt6vectorI21AdditionalModelOutputEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setGatherGenerationLogitsEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig25setGatherGenerationLogitsEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig25setGatherGenerationLogitsEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setGatherGenerationLogits__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aea23feffd2cd5e988e659d6ae4c7aed1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setGatherGenerationLogits</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">gatherGenerationLogits</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setGatherGenerationLogitsEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig24setPromptTableOffloadingEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig24setPromptTableOffloadingEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig24setPromptTableOffloadingEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setPromptTableOffloading__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9ccfa19c000f2d92054b56218eabba5a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPromptTableOffloading</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">promptTableOffloading</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig24setPromptTableOffloadingEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setCacheTransceiverConfigERK22CacheTransceiverConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig25setCacheTransceiverConfigERK22CacheTransceiverConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig25setCacheTransceiverConfigERK22CacheTransceiverConfig"></span><span id="tensorrt_llm::executor::ExecutorConfig::setCacheTransceiverConfig__CacheTransceiverConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a83ed632d68c06aa574b33238f0c75eab"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCacheTransceiverConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">cacheTransceiverConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setCacheTransceiverConfigERK22CacheTransceiverConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setEnableTrtOverlapEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig19setEnableTrtOverlapEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig19setEnableTrtOverlapEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setEnableTrtOverlap__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9b2ed3d50d5e03d614535f12d1ee8a52"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEnableTrtOverlap</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableTrtOverlap</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setEnableTrtOverlapEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig36setFailFastOnAttentionWindowTooLargeEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig36setFailFastOnAttentionWindowTooLargeEb"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig36setFailFastOnAttentionWindowTooLargeEb"></span><span id="tensorrt_llm::executor::ExecutorConfig::setFailFastOnAttentionWindowTooLarge__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9a6c5db91eb0ac42f5c25f856d3122b2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFailFastOnAttentionWindowTooLarge</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">failFastOnAttentionWindowTooLarge</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig36setFailFastOnAttentionWindowTooLargeEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::kDefaultMaxSeqIdleMicroseconds__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4cb2fb0a75c587a97ceabfb7556bb4f1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxSeqIdleMicroseconds</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">duration_cast</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">microseconds</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">(</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">minutes</span></span><span class="p"><span class="pre">(</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">.</span></span><span class="n"><span class="pre">count</span></span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::kDefaultIterStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a85af795ceffac779ed4b9e072905351f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultIterStatsMaxIterations</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1000</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::kDefaultRequestStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a4f8c59a591914a7df498b1964606bb57"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultRequestStatsMaxIterations</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBeamWidthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig13mMaxBeamWidthE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig13mMaxBeamWidthE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mMaxBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a94c48a5d5ec190006f387c7a2b6704a8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBeamWidth</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBeamWidthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The beam width value of requests that will be sent to the executor. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mSchedulerConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig16mSchedulerConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig16mSchedulerConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mSchedulerConfig__SchedulerConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ab1625bee6fca601abb5f1eb6b6355b5b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSchedulerConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mSchedulerConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The scheduler configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig14mKvCacheConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig14mKvCacheConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig14mKvCacheConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mKvCacheConfig__KvCacheConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a9c032445833352f9d7878d16a145ed4f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="tensorrt_llm::executor::KvCacheConfig"><span class="n"><span class="pre">KvCacheConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mKvCacheConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14mKvCacheConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The KV cache configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mEnableChunkedContextE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig21mEnableChunkedContextE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig21mEnableChunkedContextE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mEnableChunkedContext__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a82112a8e221adc965968a1f127ba473e"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableChunkedContext</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mEnableChunkedContextE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls whether context is allowed to be chunked. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mNormalizeLogProbsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig18mNormalizeLogProbsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig18mNormalizeLogProbsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mNormalizeLogProbs__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a84ac3e0d9136d145a47d24f5eef306c3"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNormalizeLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mNormalizeLogProbsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if log probabilities should be normalized or not. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mIterStatsMaxIterationsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23mIterStatsMaxIterationsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23mIterStatsMaxIterationsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mIterStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a201b184dee2e18455e22f45e645ac113"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIterStatsMaxIterations</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mIterStatsMaxIterationsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the maximum number of iterations for which to keep statistics. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mRequestStatsMaxIterationsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig26mRequestStatsMaxIterationsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig26mRequestStatsMaxIterationsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mRequestStatsMaxIterations__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a185a12d03febdc22bf56aedb1b05a80b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRequestStatsMaxIterations</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mRequestStatsMaxIterationsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the maximum number of iterations for which to keep per-request statistics. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mBatchingTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig13mBatchingTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig13mBatchingTypeE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mBatchingType__BatchingType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a59f638d7780b319b27c606352db470d3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE" title="tensorrt_llm::executor::BatchingType"><span class="n"><span class="pre">BatchingType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBatchingType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mBatchingTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The type of batching strategy to use. See BatchingType. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBatchSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig13mMaxBatchSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig13mMaxBatchSizeE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mMaxBatchSize__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a02960bc64feef914f78fc0712796535f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBatchSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBatchSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The max batch size of requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxNumTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig13mMaxNumTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig13mMaxNumTokensE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mMaxNumTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a0b08ae45ad7fadece12e0112f0a6e06b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxNumTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxNumTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The max number of tokens per batch. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mParallelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15mParallelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15mParallelConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mParallelConfig__std::optional:ParallelConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a723466ec6c98eb16b4e8afc69092702d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="tensorrt_llm::executor::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mParallelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mParallelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The parallel execution configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mPeftCacheConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig16mPeftCacheConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig16mPeftCacheConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mPeftCacheConfig__std::optional:PeftCacheConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aa36980dc3db09507bf58db6d5323294d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPeftCacheConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mPeftCacheConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mLogitsPostProcessorConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig26mLogitsPostProcessorConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig26mLogitsPostProcessorConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mLogitsPostProcessorConfig__std::optional:LogitsPostProcessorConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a969008c97bd9bdc4d054007a7fc84556"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE" title="tensorrt_llm::executor::LogitsPostProcessorConfig"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLogitsPostProcessorConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mLogitsPostProcessorConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Logits post processor configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig15mDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig15mDecodingConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mDecodingConfig__std::optional:DecodingConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a0861d3ae711ae600f5f6596128990a66"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Decoding configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig20mUseGpuDirectStorageE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig20mUseGpuDirectStorageE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig20mUseGpuDirectStorageE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mUseGpuDirectStorage__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aa61276fb469723282fccb160909bd60d"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseGpuDirectStorage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20mUseGpuDirectStorageE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enable/disable use of GPU Direct Storage (GDS) to load engines. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mGpuWeightsPercentE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig18mGpuWeightsPercentE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig18mGpuWeightsPercentE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mGpuWeightsPercent__float"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a418f075a4e8f30f52c45cbb6cffec2d5"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGpuWeightsPercent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mGpuWeightsPercentE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>GPU weights percent for weight streaming. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxQueueSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig13mMaxQueueSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig13mMaxQueueSizeE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mMaxQueueSize__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a90656867544080bda93c0fac2739fafd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxQueueSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxQueueSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The maximum number of requests allowed in queue before rejecting new requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig30mExtendedRuntimePerfKnobConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig30mExtendedRuntimePerfKnobConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig30mExtendedRuntimePerfKnobConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mExtendedRuntimePerfKnobConfig__ExtendedRuntimePerfKnobConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ab97f65317c339699f4cf2ee795aff3f4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mExtendedRuntimePerfKnobConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30mExtendedRuntimePerfKnobConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Config for perf knobs that can be set in runtime. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig12mDebugConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig12mDebugConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig12mDebugConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mDebugConfig__std::optional:DebugConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a3b38d197adb1142cd08a9f22c3b20378"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE" title="tensorrt_llm::executor::DebugConfig"><span class="n"><span class="pre">DebugConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDebugConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig12mDebugConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Debugging configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mRecvPollPeriodMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig17mRecvPollPeriodMsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig17mRecvPollPeriodMsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mRecvPollPeriodMs__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aedf0da0e27422613d74b042a30fdcfe0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRecvPollPeriodMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mRecvPollPeriodMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The time in ms between polls for new communication in orchestrator mode. Use 0 for busy loop. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mMaxSeqIdleMicrosecondsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23mMaxSeqIdleMicrosecondsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23mMaxSeqIdleMicrosecondsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mMaxSeqIdleMicroseconds__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a21914eda2bab89712790d945d31d0383"></span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxSeqIdleMicroseconds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mMaxSeqIdleMicrosecondsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The maximum time in microseconds a scheduled request can remain idle before getting terminated. Default value is 3 minutes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mSpeculativeDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig26mSpeculativeDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig26mSpeculativeDecodingConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mSpeculativeDecodingConfig__std::optional:SpeculativeDecodingConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ac0e289586d6a768ff070a8c2c16645ad"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSpeculativeDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mSpeculativeDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The speculative decoding configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mGuidedDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig21mGuidedDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig21mGuidedDecodingConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mGuidedDecodingConfig__std::optional:GuidedDecodingConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1acfdc8fbac268be88a5df41cd93f68b94"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGuidedDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mGuidedDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The guided decoding configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mAdditionalModelOutputsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23mAdditionalModelOutputsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23mAdditionalModelOutputsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mAdditionalModelOutputs__std::optional:std::vector:AdditionalModelOutput::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a5a03fda5fe93a996ae7be71c996e6fd8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAdditionalModelOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mAdditionalModelOutputsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The additional outputs to gather from the model. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mCacheTransceiverConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23mCacheTransceiverConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23mCacheTransceiverConfigE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mCacheTransceiverConfig__std::optional:CacheTransceiverConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aa326fc590b10734619c248865613f95c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE" title="tensorrt_llm::executor::CacheTransceiverConfig"><span class="n"><span class="pre">CacheTransceiverConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCacheTransceiverConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mCacheTransceiverConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The cache transceiver configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mGatherGenerationLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig23mGatherGenerationLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig23mGatherGenerationLogitsE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mGatherGenerationLogits__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a663d55b9741e3e75c3f821a20c27c223"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGatherGenerationLogits</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mGatherGenerationLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if generation logits should be gathered, so that returnGenerationLogits can be requested. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig22mPromptTableOffloadingE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig22mPromptTableOffloadingE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig22mPromptTableOffloadingE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mPromptTableOffloading__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1ad28f9dd9fdb366e7c58c452f8775e9ac"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPromptTableOffloading</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig22mPromptTableOffloadingE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if prompt table offloading is enabled. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mEnableTrtOverlapE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig17mEnableTrtOverlapE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig17mEnableTrtOverlapE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mEnableTrtOverlap__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1a2142a03c0de1eef70ec816ffada1f4fc"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableTrtOverlap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mEnableTrtOverlapE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls whether preparation and TRT engine execution should be overlapped. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ExecutorConfig34mFailFastOnAttentionWindowTooLargeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ExecutorConfig34mFailFastOnAttentionWindowTooLargeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ExecutorConfig34mFailFastOnAttentionWindowTooLargeE"></span><span id="tensorrt_llm::executor::ExecutorConfig::mFailFastOnAttentionWindowTooLarge__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExecutorConfig_1aa8cb203c2c460b1ab6e614837858c226"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFailFastOnAttentionWindowTooLarge</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig34mFailFastOnAttentionWindowTooLargeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls whether to fail fast when attention window is too large to fit even a single sequence in the KV cache. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for the runtime perf knobs. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig29ExtendedRuntimePerfKnobConfigEbbb10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig29ExtendedRuntimePerfKnobConfigEbbb10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig29ExtendedRuntimePerfKnobConfigEbbb10SizeType32"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::ExtendedRuntimePerfKnobConfig__b.b.b.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1ac0c7ab9509bc9ddc4b26d51385170c15"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multiBlockMode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableContextFMHAFP32Acc</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">cudaGraphMode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cudaGraphCacheSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig29ExtendedRuntimePerfKnobConfigEbbb10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigeqERK29ExtendedRuntimePerfKnobConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigeqERK29ExtendedRuntimePerfKnobConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigeqERK29ExtendedRuntimePerfKnobConfig"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::eq-operator__ExtendedRuntimePerfKnobConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1ad70b3fe8357913ce8033d452caae3c4f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE" title="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig"><span class="n"><span class="pre">ExtendedRuntimePerfKnobConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigeqERK29ExtendedRuntimePerfKnobConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17getMultiBlockModeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17getMultiBlockModeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17getMultiBlockModeEv"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::getMultiBlockModeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1ac2d433f7f25ebab808b222785e9959ac"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultiBlockMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17getMultiBlockModeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27getEnableContextFMHAFP32AccEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27getEnableContextFMHAFP32AccEv"></span><span id="_CPPv2NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27getEnableContextFMHAFP32AccEv"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::getEnableContextFMHAFP32AccC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1aaabc1b1388ee0a1332e2336f38f778ea"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableContextFMHAFP32Acc</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27getEnableContextFMHAFP32AccEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16getCudaGraphModeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16getCudaGraphModeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16getCudaGraphModeEv"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::getCudaGraphModeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a30839851a4c06dc9a3444f46dea2ba59"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCudaGraphMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16getCudaGraphModeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21getCudaGraphCacheSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21getCudaGraphCacheSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21getCudaGraphCacheSizeEv"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::getCudaGraphCacheSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a96c4d3c54bac53bf6d362c575c198a52"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCudaGraphCacheSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21getCudaGraphCacheSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17setMultiBlockModeEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17setMultiBlockModeEb"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17setMultiBlockModeEb"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::setMultiBlockMode__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a68d48578b7b2a0dae5ca6e040580ae6e"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMultiBlockMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multiBlockMode</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17setMultiBlockModeEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27setEnableContextFMHAFP32AccEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27setEnableContextFMHAFP32AccEb"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27setEnableContextFMHAFP32AccEb"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::setEnableContextFMHAFP32Acc__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1af6f642b8d52ac9d7a7ebb308b2f52585"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEnableContextFMHAFP32Acc</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableContextFMHAFP32Acc</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27setEnableContextFMHAFP32AccEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16setCudaGraphModeEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16setCudaGraphModeEb"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16setCudaGraphModeEb"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::setCudaGraphMode__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a0a6a0d8f7f9ee18479096186a67dccb1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCudaGraphMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">cudaGraphMode</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16setCudaGraphModeEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21setCudaGraphCacheSizeE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21setCudaGraphCacheSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21setCudaGraphCacheSizeE10SizeType32"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::setCudaGraphCacheSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1aa9db12e2afad4881208fe5ccadc5ec0f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCudaGraphCacheSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cacheSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21setCudaGraphCacheSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig15mMultiBlockModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig15mMultiBlockModeE"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig15mMultiBlockModeE"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::mMultiBlockMode__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a40f5a30741838e416395fba820a111cd"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMultiBlockMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig15mMultiBlockModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Control if multi block mode should be enabled or not. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig25mEnableContextFMHAFP32AccE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig25mEnableContextFMHAFP32AccE"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig25mEnableContextFMHAFP32AccE"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::mEnableContextFMHAFP32Acc__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1abec03437378a48e9fe254b0529c9a2df"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableContextFMHAFP32Acc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig25mEnableContextFMHAFP32AccE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If enable FMHA runner FP32 accumulation. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig14mCudaGraphModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig14mCudaGraphModeE"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig14mCudaGraphModeE"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::mCudaGraphMode__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1affd41e6596e980c0f37337e91ef7a6b6"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCudaGraphMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig14mCudaGraphModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Control if enable cuda graph. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig19mCudaGraphCacheSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig19mCudaGraphCacheSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig19mCudaGraphCacheSizeE"></span><span id="tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig::mCudaGraphCacheSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExtendedRuntimePerfKnobConfig_1a944b111b52935ea2ce1054704d8c0c23"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCudaGraphCacheSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig19mCudaGraphCacheSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of cuda graphs to be cached in the runtime. The larger the cache, the better the perf, but more GPU memory is consumed. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfigE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for speculative decoding with external draft tokens. Allows to include draft tokens, draft logits and specify acceptance threshold. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig25ExternalDraftTokensConfigE9VecTokensNSt8optionalI6TensorEERKNSt8optionalI9FloatTypeEERKNSt8optionalIbEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfig25ExternalDraftTokensConfigE9VecTokensNSt8optionalI6TensorEERKNSt8optionalI9FloatTypeEERKNSt8optionalIbEE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfig25ExternalDraftTokensConfigE9VecTokensNSt8optionalI6TensorEERKNSt8optionalI9FloatTypeEERKNSt8optionalIbEE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::ExternalDraftTokensConfig__VecTokens.std::optional:Tensor:.std::optional:FloatType:CR.std::optional:b:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1a81c077472035062e1e68034d1e762635"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">logits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">acceptanceThreshold</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">fastLogits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig25ExternalDraftTokensConfigE9VecTokensNSt8optionalI6TensorEERKNSt8optionalI9FloatTypeEERKNSt8optionalIbEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getTokensEv"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::getTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1ad99090a218772e934a91558571ea4f67"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getLogitsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getLogitsEv"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::getLogitsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1a23d59333fbfd34696f1f9ef1e386182d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getLogitsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig22getAcceptanceThresholdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25ExternalDraftTokensConfig22getAcceptanceThresholdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25ExternalDraftTokensConfig22getAcceptanceThresholdEv"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::getAcceptanceThresholdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1a179d3620d160c642b515b48c6ccaea22"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAcceptanceThreshold</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig22getAcceptanceThresholdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig13getFastLogitsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25ExternalDraftTokensConfig13getFastLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25ExternalDraftTokensConfig13getFastLogitsEv"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::getFastLogitsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1ad0eae155e1cc3957e6edd3226b430ca9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFastLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig13getFastLogitsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfig7mTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfig7mTokensE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::mTokens__VecTokens"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1a648382f7b513a2fb82330540687833e3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The draft tokens. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfig7mLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfig7mLogitsE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::mLogits__std::optional:Tensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1add7e98a70f09f15c3390a39a7673b5a5"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The draft logits. Expected shape: [num_draft_tokens, vocab_size]. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig20mAcceptanceThresholdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfig20mAcceptanceThresholdE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfig20mAcceptanceThresholdE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::mAcceptanceThreshold__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1abcc619153ad5a5e2c5bf08940eda9064"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAcceptanceThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig20mAcceptanceThresholdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The acceptance threshold. Must be > 0.f and <= 1.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig11mFastLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25ExternalDraftTokensConfig11mFastLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor25ExternalDraftTokensConfig11mFastLogitsE"></span><span id="tensorrt_llm::executor::ExternalDraftTokensConfig::mFastLogits__std::optional:b:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ExternalDraftTokensConfig_1a799828aa3a4ecf95517f5f7227cef430"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFastLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig11mFastLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Use direct transfer for draft logits. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfigE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Guided decoding configurations for executor. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a8a09e91495919291c648a0ef8c53d912"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuidedDecodingBackend</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend9kXGRAMMARE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend9kXGRAMMARE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend9kXGRAMMARE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a8a09e91495919291c648a0ef8c53d912af9ef3c75182d09e9a4f80fee7ae4d3e6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kXGRAMMAR</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend9kXGRAMMARE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enable guided decoding with XGrammar backend. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend11kLLGUIDANCEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend11kLLGUIDANCEE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend11kLLGUIDANCEE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a8a09e91495919291c648a0ef8c53d912ac2a7f8385259c19055da1af0f2b11bec"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLLGUIDANCE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend11kLLGUIDANCEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Enable guided decoding with LLGuidance backend. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig20GuidedDecodingConfigE21GuidedDecodingBackendNSt8optionalINSt6vectorINSt6stringEEEEENSt8optionalINSt6stringEEENSt8optionalINSt6vectorI11TokenIdTypeEEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig20GuidedDecodingConfigE21GuidedDecodingBackendNSt8optionalINSt6vectorINSt6stringEEEEENSt8optionalINSt6stringEEENSt8optionalINSt6vectorI11TokenIdTypeEEEE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig20GuidedDecodingConfigE21GuidedDecodingBackendNSt8optionalINSt6vectorINSt6stringEEEEENSt8optionalINSt6stringEEENSt8optionalINSt6vectorI11TokenIdTypeEEEE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::GuidedDecodingConfig__GuidedDecodingBackend.std::optional:std::vector:ss::.std::optional:ss:.std::optional:std::vector:TokenIdType::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a9d6fc21aebb2c45bf09d150249f10683"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE" title="tensorrt_llm::executor::GuidedDecodingConfig::GuidedDecodingBackend"><span class="n"><span class="pre">GuidedDecodingBackend</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">backend</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">encodedVocab</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tokenizerStr</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stopTokenIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig20GuidedDecodingConfigE21GuidedDecodingBackendNSt8optionalINSt6vectorINSt6stringEEEEENSt8optionalINSt6stringEEENSt8optionalINSt6vectorI11TokenIdTypeEEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfigeqERK20GuidedDecodingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfigeqERK20GuidedDecodingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfigeqERK20GuidedDecodingConfig"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::eq-operator__GuidedDecodingConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a71e9e9dffeefe7149c0859c82b984ece"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE" title="tensorrt_llm::executor::GuidedDecodingConfig"><span class="n"><span class="pre">GuidedDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfigeqERK20GuidedDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig10setBackendERK21GuidedDecodingBackend">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig10setBackendERK21GuidedDecodingBackend"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig10setBackendERK21GuidedDecodingBackend"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::setBackend__GuidedDecodingBackendCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1ab79112af809cfce8fa5ea720f8eb2551"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBackend</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE" title="tensorrt_llm::executor::GuidedDecodingConfig::GuidedDecodingBackend"><span class="n"><span class="pre">GuidedDecodingBackend</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">backend</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig10setBackendERK21GuidedDecodingBackend" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig10getBackendEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfig10getBackendEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfig10getBackendEv"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::getBackendC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1aacf9c80ebd33edb32b34f354210e9b12"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE" title="tensorrt_llm::executor::GuidedDecodingConfig::GuidedDecodingBackend"><span class="n"><span class="pre">GuidedDecodingBackend</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBackend</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig10getBackendEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setEncodedVocabERKNSt6vectorINSt6stringEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig15setEncodedVocabERKNSt6vectorINSt6stringEEE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig15setEncodedVocabERKNSt6vectorINSt6stringEEE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::setEncodedVocab__std::vector:ss:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1ac33775dbefa1b6f763114d88f8a9cd0b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEncodedVocab</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">encodedVocab</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setEncodedVocabERKNSt6vectorINSt6stringEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getEncodedVocabEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfig15getEncodedVocabEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfig15getEncodedVocabEv"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::getEncodedVocabC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a59c96bec37ce22abbef8f8a3188350fc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEncodedVocab</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getEncodedVocabEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setTokenizerStrERKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig15setTokenizerStrERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig15setTokenizerStrERKNSt6stringE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::setTokenizerStr__ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1aa0d470301f6e1dfff0f61c8fb45d8c58"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTokenizerStr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tokenizerStr</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setTokenizerStrERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getTokenizerStrEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfig15getTokenizerStrEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfig15getTokenizerStrEv"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::getTokenizerStrC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a126b7221f9d2d73e2eafe6c587439b68"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTokenizerStr</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getTokenizerStrEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setStopTokenIdsERKNSt6vectorI11TokenIdTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig15setStopTokenIdsERKNSt6vectorI11TokenIdTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig15setStopTokenIdsERKNSt6vectorI11TokenIdTypeEE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::setStopTokenIds__std::vector:TokenIdType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a23a578a4affc585b4bac7c41f15d34e9"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setStopTokenIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">stopTokenIds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setStopTokenIdsERKNSt6vectorI11TokenIdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getStopTokenIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfig15getStopTokenIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfig15getStopTokenIdsEv"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::getStopTokenIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a7dd0f7e137c5f31e34eb1bccf60d188f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getStopTokenIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getStopTokenIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig8validateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingConfig8validateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingConfig8validateEv"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::validateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1aa069dcb5d299be6720d2ccb3929fc0a1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">validate</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig8validateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig8mBackendE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig8mBackendE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig8mBackendE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::mBackend__GuidedDecodingBackend"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1afc33e7f31234b08feb15f93b858e3c83"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE" title="tensorrt_llm::executor::GuidedDecodingConfig::GuidedDecodingBackend"><span class="n"><span class="pre">GuidedDecodingBackend</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBackend</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig8mBackendE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Guided decoding backend. Currently supports XGrammar. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mEncodedVocabE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig13mEncodedVocabE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig13mEncodedVocabE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::mEncodedVocab__std::optional:std::vector:ss::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1af2615dccce382e26f070b3afb89ff3c7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEncodedVocab</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mEncodedVocabE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Encoded vocabulary. For a huggingface tokenizer, it can be extracted by: </p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">encoded_vocab</span> <span class="o">=</span> <span class="n">tokenizer</span><span class="o">.</span><span class="n">get_vocab</span><span class="p">()</span>
|
||
<span class="n">encoded_vocab</span> <span class="o">=</span> <span class="p">[</span><span class="n">token</span> <span class="k">for</span> <span class="n">token</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">encoded_vocab</span><span class="o">.</span><span class="n">items</span><span class="p">(),</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">])]</span>
|
||
</pre></div>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mTokenizerStrE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig13mTokenizerStrE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig13mTokenizerStrE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::mTokenizerStr__std::optional:ss:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1adc4800cf6648cfa71528276372de8197"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTokenizerStr</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mTokenizerStrE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Tokenizer string. For a huggingface fast tokenizer, it can be extracted by: </p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">tokenizer_str</span> <span class="o">=</span> <span class="n">tokenizer</span><span class="o">.</span><span class="n">backend_tokenizer</span><span class="o">.</span><span class="n">to_str</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mStopTokenIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingConfig13mStopTokenIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingConfig13mStopTokenIdsE"></span><span id="tensorrt_llm::executor::GuidedDecodingConfig::mStopTokenIds__std::optional:std::vector:TokenIdType::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingConfig_1a405c8a802aadfaa66102519c00c9be33"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mStopTokenIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mStopTokenIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Stop token ids. If not provided, it can be automatically detected. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParamsE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParamsE"></span><span id="tensorrt_llm::executor::GuidedDecodingParams"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuidedDecodingParams</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Guided decoding parameters for a request. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuideType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType5kJSONE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideType5kJSONE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideType5kJSONE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946a7820f24be07e92b4ed8ef25f129cf422"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kJSON</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType5kJSONE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generated text is amenable to json format. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType12kJSON_SCHEMAE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideType12kJSON_SCHEMAE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideType12kJSON_SCHEMAE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946a08c4ca8b86b0f06177c4d0b7041e938f"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kJSON_SCHEMA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType12kJSON_SCHEMAE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generated text is amenable to json format with additional user-specified restrictions, namely schema. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType6kREGEXE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideType6kREGEXE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideType6kREGEXE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946ae6485c71429ab44931687b52af697e4f"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kREGEX</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType6kREGEXE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generated text is amenable to the user-specified regular expression. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType13kEBNF_GRAMMARE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideType13kEBNF_GRAMMARE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideType13kEBNF_GRAMMARE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946ae71f648967026c8d1d393580be763498"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEBNF_GRAMMAR</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType13kEBNF_GRAMMARE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generated text is amenable to the user-specified extended Backus-Naur form (EBNF) grammar. EBNF grammar is widely-used to express context-free grammars. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType15kSTRUCTURAL_TAGE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams9GuideType15kSTRUCTURAL_TAGE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams9GuideType15kSTRUCTURAL_TAGE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a3e79896af4d977fc8eedb6dae21c5946a8d2b72d3ee9b5a4abfe77de4dee3962c"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kSTRUCTURAL_TAG</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType15kSTRUCTURAL_TAGE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generated text is amenable to the XGrammar structural tag. </p>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams20GuidedDecodingParamsE9GuideTypeNSt8optionalINSt6stringEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams20GuidedDecodingParamsE9GuideTypeNSt8optionalINSt6stringEEE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams20GuidedDecodingParamsE9GuideTypeNSt8optionalINSt6stringEEE"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::GuidedDecodingParams__GuideType.std::optional:ss:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a6ca5a46030ce345f2d0c61430af3268d"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GuidedDecodingParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE" title="tensorrt_llm::executor::GuidedDecodingParams::GuideType"><span class="n"><span class="pre">GuideType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">guideType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">guide</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams20GuidedDecodingParamsE9GuideTypeNSt8optionalINSt6stringEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParamseqERK20GuidedDecodingParams">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingParamseqERK20GuidedDecodingParams"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingParamseqERK20GuidedDecodingParams"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::eq-operator__GuidedDecodingParamsCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a9895d4a2338b7200a8be03babbd61eaf"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParamseqERK20GuidedDecodingParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams12getGuideTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingParams12getGuideTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingParams12getGuideTypeEv"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::getGuideTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a11a3c820302df3f1c548a07a040a5c59"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE" title="tensorrt_llm::executor::GuidedDecodingParams::GuideType"><span class="n"><span class="pre">GuideType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGuideType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams12getGuideTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams8getGuideEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20GuidedDecodingParams8getGuideEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20GuidedDecodingParams8getGuideEv"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::getGuideC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a86814394d79a4e7550e3faa7d040bb74"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGuide</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams8getGuideEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams10mGuideTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams10mGuideTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams10mGuideTypeE"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::mGuideType__GuideType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1ac2f74b2d1059b890383b2225bc0a7c53"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE" title="tensorrt_llm::executor::GuidedDecodingParams::GuideType"><span class="n"><span class="pre">GuideType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGuideType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams10mGuideTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The guide type. See GuideType. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams6mGuideE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20GuidedDecodingParams6mGuideE"></span><span id="_CPPv2N12tensorrt_llm8executor20GuidedDecodingParams6mGuideE"></span><span id="tensorrt_llm::executor::GuidedDecodingParams::mGuide__std::optional:ss:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1GuidedDecodingParams_1a80cc2ed848932eed934f79a1f5cdcc8e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGuide</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams6mGuideE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The detailed guide string. It could be a json schema, a regular expression or a EBNF grammar depending on mGuideType. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17JsonSerializationE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17JsonSerializationE"></span><span id="_CPPv2N12tensorrt_llm8executor17JsonSerializationE"></span><span id="tensorrt_llm::executor::JsonSerialization"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1JsonSerialization"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">JsonSerialization</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17JsonSerializationE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Class with utility functions to serialize statistics to json string. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK14IterationStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK14IterationStats"></span><span id="_CPPv2N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK14IterationStats"></span><span id="tensorrt_llm::executor::JsonSerialization::toJsonStr__IterationStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1JsonSerialization_1abb7c3366de8460accfc20764a1c820c6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toJsonStr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE" title="tensorrt_llm::executor::IterationStats"><span class="n"><span class="pre">IterationStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">iterationStats</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK14IterationStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Utility function to convert an iterationStats struct to a json serialized string. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK24RequestStatsPerIteration">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK24RequestStatsPerIteration"></span><span id="_CPPv2N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK24RequestStatsPerIteration"></span><span id="tensorrt_llm::executor::JsonSerialization::toJsonStr__RequestStatsPerIterationCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1JsonSerialization_1a8734846c416a9d9199d72c0fc6429374"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toJsonStr</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE" title="tensorrt_llm::executor::RequestStatsPerIteration"><span class="n"><span class="pre">RequestStatsPerIteration</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestStatsPerIter</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK24RequestStatsPerIteration" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Utility function to convert a requestStatsPerIteration struct to a json serialized string. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK12RequestStats">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK12RequestStats"></span><span id="_CPPv2N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK12RequestStats"></span><span id="tensorrt_llm::executor::JsonSerialization::toJsonStr__RequestStatsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1JsonSerialization_1a7cbb6808bdbace0121a04044ec044854"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toJsonStr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE" title="tensorrt_llm::executor::RequestStats"><span class="n"><span class="pre">RequestStats</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestStats</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK12RequestStats" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Utility function to convert a requestStats struct to a json serialized string. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfigE"></span><span id="tensorrt_llm::executor::KvCacheConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for the KV cache. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig13KvCacheConfigEbRKNSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI6size_tEEbRKNSt8optionalI9FloatTypeEENSt8optionalI17RetentionPriorityEE6size_tbbb10SizeType32RKNSt8optionalIN12tensorrt_llm7runtime15RuntimeDefaultsEEERK8uint64_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig13KvCacheConfigEbRKNSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI6size_tEEbRKNSt8optionalI9FloatTypeEENSt8optionalI17RetentionPriorityEE6size_tbbb10SizeType32RKNSt8optionalIN12tensorrt_llm7runtime15RuntimeDefaultsEEERK8uint64_t"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig13KvCacheConfigEbRKNSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI6size_tEEbRKNSt8optionalI9FloatTypeEENSt8optionalI17RetentionPriorityEE6size_tbbb10SizeType32RKNSt8optionalIN12tensorrt_llm7runtime15RuntimeDefaultsEEERK8uint64_t"></span><span id="tensorrt_llm::executor::KvCacheConfig::KvCacheConfig__b.std::optional:SizeType32:CR.std::optional:std::vector:SizeType32::CR.std::optional:SizeType32:CR.std::optional:FloatType:CR.std::optional:s:CR.b.std::optional:FloatType:CR.std::optional:RetentionPriority:.s.b.b.b.SizeType32.std::optional:tensorrt_llm::runtime::RuntimeDefaults:CR.uint64_tCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a7e3eb8b0adc2bef29614fa9a314fb801"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBlockReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">maxTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">maxAttentionWindowVec</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">freeGpuMemoryFraction</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">hostCacheSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">onboardBlocks</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">crossKvCacheFraction</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">secondaryOffloadMinPriority</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">eventBufferMaxSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enablePartialReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">copyOnPartialReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useUvm</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionDpEventsGatherPeriodMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="tensorrt_llm::runtime::RuntimeDefaults"><span class="n"><span class="pre">RuntimeDefaults</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">runtimeDefaults</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">maxGpuTotalBytes</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig13KvCacheConfigEbRKNSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI6size_tEEbRKNSt8optionalI9FloatTypeEENSt8optionalI17RetentionPriorityEE6size_tbbb10SizeType32RKNSt8optionalIN12tensorrt_llm7runtime15RuntimeDefaultsEEERK8uint64_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getEnableBlockReuseEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig19getEnableBlockReuseEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig19getEnableBlockReuseEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getEnableBlockReuseC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a215347e4303538281e8918d711bee0fe"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableBlockReuse</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getEnableBlockReuseEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEnablePartialReuseEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig21getEnablePartialReuseEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig21getEnablePartialReuseEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getEnablePartialReuseC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a56a69c642f4b2cf2d8028d34ded8092a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnablePartialReuse</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEnablePartialReuseEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getCopyOnPartialReuseEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig21getCopyOnPartialReuseEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig21getCopyOnPartialReuseEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getCopyOnPartialReuseC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ae7b44cbffea938c00965637c898fe885"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCopyOnPartialReuse</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getCopyOnPartialReuseEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig12getMaxTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig12getMaxTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig12getMaxTokensEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getMaxTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a8f8a9fc6161d9f540cdd41e568e6703d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig12getMaxTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getMaxAttentionWindowVecEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig24getMaxAttentionWindowVecEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig24getMaxAttentionWindowVecEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getMaxAttentionWindowVecC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ab8f28550f538ac770340947ebb99a0aa"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxAttentionWindowVec</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getMaxAttentionWindowVecEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig18getSinkTokenLengthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig18getSinkTokenLengthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig18getSinkTokenLengthEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getSinkTokenLengthC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ad63d1e7cc46b30f38f2f7811419f2de0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSinkTokenLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig18getSinkTokenLengthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getFreeGpuMemoryFractionEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig24getFreeGpuMemoryFractionEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig24getFreeGpuMemoryFractionEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getFreeGpuMemoryFractionC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ac216ec0143b80ed2538d54e27ad632d7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFreeGpuMemoryFraction</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getFreeGpuMemoryFractionEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig23getCrossKvCacheFractionEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig23getCrossKvCacheFractionEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig23getCrossKvCacheFractionEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getCrossKvCacheFractionC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a57844d3da7f5bfdf467a7f76b5c88278"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCrossKvCacheFraction</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig23getCrossKvCacheFractionEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getHostCacheSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig16getHostCacheSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig16getHostCacheSizeEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getHostCacheSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a12ce86b6f1a03e50d65462f94a6f90df"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getHostCacheSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getHostCacheSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getOnboardBlocksEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig16getOnboardBlocksEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig16getOnboardBlocksEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getOnboardBlocksC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a596dc5ef42e0aaeffa728705972635f5"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOnboardBlocks</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getOnboardBlocksEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig30getSecondaryOffloadMinPriorityEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig30getSecondaryOffloadMinPriorityEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig30getSecondaryOffloadMinPriorityEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getSecondaryOffloadMinPriorityC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1adc7ee22b315728061c46b3936a1bc790"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSecondaryOffloadMinPriority</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig30getSecondaryOffloadMinPriorityEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEventBufferMaxSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig21getEventBufferMaxSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig21getEventBufferMaxSizeEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getEventBufferMaxSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1afff662f63bfe3292d951951884aee2f2"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEventBufferMaxSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEventBufferMaxSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig9getUseUvmEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig9getUseUvmEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig9getUseUvmEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getUseUvmC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a9b7d62d000c4beda7c474a4068c99362"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getUseUvm</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig9getUseUvmEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig34getAttentionDpEventsGatherPeriodMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig34getAttentionDpEventsGatherPeriodMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig34getAttentionDpEventsGatherPeriodMsEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getAttentionDpEventsGatherPeriodMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ab835a2e755283f286d4456acff6a47c4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAttentionDpEventsGatherPeriodMs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig34getAttentionDpEventsGatherPeriodMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getMaxGpuTotalBytesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor13KvCacheConfig19getMaxGpuTotalBytesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor13KvCacheConfig19getMaxGpuTotalBytesEv"></span><span id="tensorrt_llm::executor::KvCacheConfig::getMaxGpuTotalBytesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a5775c28425230c6945f17546eed2c829"></span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxGpuTotalBytes</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getMaxGpuTotalBytesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setEnableBlockReuseEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig19setEnableBlockReuseEb"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig19setEnableBlockReuseEb"></span><span id="tensorrt_llm::executor::KvCacheConfig::setEnableBlockReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a934f7362c2d931489c7df5c86a5c19c1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEnableBlockReuse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBlockReuse</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setEnableBlockReuseEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEnablePartialReuseEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig21setEnablePartialReuseEb"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig21setEnablePartialReuseEb"></span><span id="tensorrt_llm::executor::KvCacheConfig::setEnablePartialReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a511b882ee5fde27ad4b1079c72228d5a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEnablePartialReuse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enablePartialReuse</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEnablePartialReuseEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setCopyOnPartialReuseEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig21setCopyOnPartialReuseEb"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig21setCopyOnPartialReuseEb"></span><span id="tensorrt_llm::executor::KvCacheConfig::setCopyOnPartialReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a761ee952221742cc8453425ecb934973"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCopyOnPartialReuse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">copyOnPartialReuse</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setCopyOnPartialReuseEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig12setMaxTokensENSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig12setMaxTokensENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig12setMaxTokensENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::KvCacheConfig::setMaxTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1abf759c98e8b13142e7c9342d044e2db1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokens</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig12setMaxTokensENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setMaxAttentionWindowVecENSt6vectorI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig24setMaxAttentionWindowVecENSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig24setMaxAttentionWindowVecENSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::executor::KvCacheConfig::setMaxAttentionWindowVec__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ad7acf1dcb06442e734a36bce18154387"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxAttentionWindowVec</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindowVec</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setMaxAttentionWindowVecENSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig18setSinkTokenLengthE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig18setSinkTokenLengthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig18setSinkTokenLengthE10SizeType32"></span><span id="tensorrt_llm::executor::KvCacheConfig::setSinkTokenLength__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1abc33d9483367d58b2744884618840a37"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSinkTokenLength</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig18setSinkTokenLengthE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setFreeGpuMemoryFractionE9FloatType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig24setFreeGpuMemoryFractionE9FloatType"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig24setFreeGpuMemoryFractionE9FloatType"></span><span id="tensorrt_llm::executor::KvCacheConfig::setFreeGpuMemoryFraction__FloatType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1acf2e6b9f0c81b3c1a299e96a6737b587"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFreeGpuMemoryFraction</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">freeGpuMemoryFraction</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setFreeGpuMemoryFractionE9FloatType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig23setCrossKvCacheFractionE9FloatType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig23setCrossKvCacheFractionE9FloatType"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig23setCrossKvCacheFractionE9FloatType"></span><span id="tensorrt_llm::executor::KvCacheConfig::setCrossKvCacheFraction__FloatType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a2985e832738877a2d522ded1d630a7ed"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCrossKvCacheFraction</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">crossKvCacheFraction</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig23setCrossKvCacheFractionE9FloatType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setHostCacheSizeE6size_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig16setHostCacheSizeE6size_t"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig16setHostCacheSizeE6size_t"></span><span id="tensorrt_llm::executor::KvCacheConfig::setHostCacheSize__s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ab23edc123b320ef59c4c4117e7c97117"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setHostCacheSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hostCacheSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setHostCacheSizeE6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setOnboardBlocksEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig16setOnboardBlocksEb"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig16setOnboardBlocksEb"></span><span id="tensorrt_llm::executor::KvCacheConfig::setOnboardBlocks__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ac1ff38e4da7ddf084bc4ad5192257193"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setOnboardBlocks</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">onboardBlocks</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setOnboardBlocksEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig30setSecondaryOffloadMinPriorityENSt8optionalI17RetentionPriorityEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig30setSecondaryOffloadMinPriorityENSt8optionalI17RetentionPriorityEE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig30setSecondaryOffloadMinPriorityENSt8optionalI17RetentionPriorityEE"></span><span id="tensorrt_llm::executor::KvCacheConfig::setSecondaryOffloadMinPriority__std::optional:RetentionPriority:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ae4dcc95f8c89af99242d43b806dd189d"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSecondaryOffloadMinPriority</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">secondaryOffloadMinPriority</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig30setSecondaryOffloadMinPriorityENSt8optionalI17RetentionPriorityEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEventBufferMaxSizeE6size_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig21setEventBufferMaxSizeE6size_t"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig21setEventBufferMaxSizeE6size_t"></span><span id="tensorrt_llm::executor::KvCacheConfig::setEventBufferMaxSize__s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1acaa651dc98ad9d1ed7ce153315bcea93"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEventBufferMaxSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">eventBufferMaxSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEventBufferMaxSizeE6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig9setUseUvmEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig9setUseUvmEb"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig9setUseUvmEb"></span><span id="tensorrt_llm::executor::KvCacheConfig::setUseUvm__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a87475c188ab07417058dd7166334662b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseUvm</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useUvm</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig9setUseUvmEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig34setAttentionDpEventsGatherPeriodMsE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig34setAttentionDpEventsGatherPeriodMsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig34setAttentionDpEventsGatherPeriodMsE10SizeType32"></span><span id="tensorrt_llm::executor::KvCacheConfig::setAttentionDpEventsGatherPeriodMs__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a094b7922050852ba020965ac2b1c5104"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setAttentionDpEventsGatherPeriodMs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionDpEventsGatherPeriodMs</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig34setAttentionDpEventsGatherPeriodMsE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setMaxGpuTotalBytesE8uint64_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig19setMaxGpuTotalBytesE8uint64_t"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig19setMaxGpuTotalBytesE8uint64_t"></span><span id="tensorrt_llm::executor::KvCacheConfig::setMaxGpuTotalBytes__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1acb7b1781ab4511769f0203e13ebe9131"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxGpuTotalBytes</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxGpuTotalBytes</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setMaxGpuTotalBytesE8uint64_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig34fillEmptyFieldsFromRuntimeDefaultsERKN12tensorrt_llm7runtime15RuntimeDefaultsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig34fillEmptyFieldsFromRuntimeDefaultsERKN12tensorrt_llm7runtime15RuntimeDefaultsE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig34fillEmptyFieldsFromRuntimeDefaultsERKN12tensorrt_llm7runtime15RuntimeDefaultsE"></span><span id="tensorrt_llm::executor::KvCacheConfig::fillEmptyFieldsFromRuntimeDefaults__tensorrt_llm::runtime::RuntimeDefaultsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ab0e5cb93a75e85e03bd662fa122701a0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fillEmptyFieldsFromRuntimeDefaults</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="tensorrt_llm::runtime::RuntimeDefaults"><span class="n"><span class="pre">RuntimeDefaults</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">runtimeDefaults</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig34fillEmptyFieldsFromRuntimeDefaultsERKN12tensorrt_llm7runtime15RuntimeDefaultsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig22kDefaultGpuMemFractionE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig22kDefaultGpuMemFractionE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig22kDefaultGpuMemFractionE"></span><span id="tensorrt_llm::executor::KvCacheConfig::kDefaultGpuMemFraction__auto"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ab1e9b2fa9ab05d17ac4b7a4686d6f4eb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultGpuMemFraction</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0.9F</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22kDefaultGpuMemFractionE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mEnableBlockReuseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig17mEnableBlockReuseE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig17mEnableBlockReuseE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mEnableBlockReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a60551af15587436a83fb1ca17a82878f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableBlockReuse</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mEnableBlockReuseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if KV cache blocks can be reused for different requests. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig10mMaxTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig10mMaxTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig10mMaxTokensE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mMaxTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a06bf9092bf1b1abedc4c256edb1be59a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig10mMaxTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The maximum number of tokens that should be stored in the KV cache If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mMaxAttentionWindowVecE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig22mMaxAttentionWindowVecE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig22mMaxAttentionWindowVecE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mMaxAttentionWindowVec__std::optional:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1aaf941396e46ebfc43d3f8908845b8d1e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxAttentionWindowVec</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mMaxAttentionWindowVecE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Size of the attention window for each sequence. Only the last mMaxAttentionWindow tokens of each sequence will be stored in the KV cache. Different layers may have different max attention window sizes. If the number of elements in mMaxAttentionWindowVec is less than the number of layers, mMaxAttentionWindowVec will be repeated multiple times to the number of layers. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig16mSinkTokenLengthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig16mSinkTokenLengthE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig16mSinkTokenLengthE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mSinkTokenLength__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a84f967e8a537c1060870714612acc0e4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSinkTokenLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16mSinkTokenLengthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Number of sink tokens (tokens to always keep in attention window) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mFreeGpuMemoryFractionE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig22mFreeGpuMemoryFractionE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig22mFreeGpuMemoryFractionE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mFreeGpuMemoryFraction__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ac48a05dd2ed2669970f12495e49c3cd1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFreeGpuMemoryFraction</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mFreeGpuMemoryFractionE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The fraction of GPU memory fraction that should be allocated for the KV cache. Default is 90%. If both mMaxTokens and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig21mCrossKvCacheFractionE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig21mCrossKvCacheFractionE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig21mCrossKvCacheFractionE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mCrossKvCacheFraction__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a331f2b5c3a2b92646a851936c1386c66"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCrossKvCacheFraction</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21mCrossKvCacheFractionE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The fraction of the KV Cache memory should be reserved for cross attention If set to p, self attention will use 1-p of KV Cache memory and cross attention will use p of KV Cache memory. Default is 50%. Should only be set when using encoder-decoder model. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mHostCacheSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig14mHostCacheSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig14mHostCacheSizeE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mHostCacheSize__std::optional:s:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a3ec456502080d1b581b3619ce46a0ba6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mHostCacheSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mHostCacheSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Size of secondary memory pool in bytes. Default is 0. Having a secondary memory pool increases KV cache block reuse potential. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mOnboardBlocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig14mOnboardBlocksE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig14mOnboardBlocksE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mOnboardBlocks__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ad16b0f0939192c1b97e6d2e905c3a1cc"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOnboardBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mOnboardBlocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls whether offloaded blocks should be onboarded back into primary memory before being reused. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig28mSecondaryOffloadMinPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig28mSecondaryOffloadMinPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig28mSecondaryOffloadMinPriorityE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mSecondaryOffloadMinPriority__std::optional:RetentionPriority:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a7aae6ab2ac9a6e1c677e88898f366a15"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSecondaryOffloadMinPriority</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig28mSecondaryOffloadMinPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Only blocks with priority > mSecondaryOfflineMinPriority can be offloaded to secondary memory. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEventBufferMaxSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig19mEventBufferMaxSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig19mEventBufferMaxSizeE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mEventBufferMaxSize__s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a6ef7d949ae48a5f5862c995d70b5f84f"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEventBufferMaxSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEventBufferMaxSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Max size of the KV cache event buffer. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEnablePartialReuseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig19mEnablePartialReuseE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig19mEnablePartialReuseE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mEnablePartialReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a5fa92b167b58c96959c63ed070baa1bf"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnablePartialReuse</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEnablePartialReuseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Whether blocks that are only partially matched can be reused. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mCopyOnPartialReuseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig19mCopyOnPartialReuseE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig19mCopyOnPartialReuseE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mCopyOnPartialReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1ae502a11597c521605bc754924571e805"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCopyOnPartialReuse</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mCopyOnPartialReuseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Whether partially matched blocks that are in use can be reused after copying them. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig7mUseUvmE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig7mUseUvmE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig7mUseUvmE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mUseUvm__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a2b410a4337a0ad1a149fc1240ee78732"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseUvm</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig7mUseUvmE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Whether to use UVM for the KV cache. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig32mAttentionDpEventsGatherPeriodMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig32mAttentionDpEventsGatherPeriodMsE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig32mAttentionDpEventsGatherPeriodMsE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mAttentionDpEventsGatherPeriodMs__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1aecd17a5acd27607d17f16d71cc4eee74"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAttentionDpEventsGatherPeriodMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig32mAttentionDpEventsGatherPeriodMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The period in milliseconds to gather attention DP events across ranks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mMaxGpuTotalBytesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor13KvCacheConfig17mMaxGpuTotalBytesE"></span><span id="_CPPv2N12tensorrt_llm8executor13KvCacheConfig17mMaxGpuTotalBytesE"></span><span id="tensorrt_llm::executor::KvCacheConfig::mMaxGpuTotalBytes__uint64_t"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheConfig_1a320eee836484ca154c43e10d26f544ef"></span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxGpuTotalBytes</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mMaxGpuTotalBytesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The maximum size in bytes of GPU memory that can be allocated for the KV cache. If both mMaxGpuTotalBytes and mFreeGpuMemoryFraction are specified, memory corresponding to the minimum will be allocated. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheCreatedDataE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheCreatedDataE"></span><span id="tensorrt_llm::executor::KVCacheCreatedData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheCreatedData"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheCreatedData</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheCreatedData22numBlocksPerCacheLevelE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheCreatedData22numBlocksPerCacheLevelE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheCreatedData22numBlocksPerCacheLevelE"></span><span id="tensorrt_llm::executor::KVCacheCreatedData::numBlocksPerCacheLevel__std::vector:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheCreatedData_1abae9e965629624bc13a99b0b4b72dcf9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numBlocksPerCacheLevel</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedData22numBlocksPerCacheLevelE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The amount of blocks at each cache level. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEventE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEventE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEventE"></span><span id="tensorrt_llm::executor::KVCacheEvent"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEvent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEvent12KVCacheEventE6IdType16KVCacheEventData10SizeType32NSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEvent12KVCacheEventE6IdType16KVCacheEventData10SizeType32NSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEvent12KVCacheEventE6IdType16KVCacheEventData10SizeType32NSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::KVCacheEvent::KVCacheEvent__IdType.KVCacheEventData.SizeType32.std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent_1a5d441b5850e2edd71642b107ca0dc002"></span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEvent</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">eventId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE" title="tensorrt_llm::executor::KVCacheEventData"><span class="n"><span class="pre">KVCacheEventData</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">attentionDpRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent12KVCacheEventE6IdType16KVCacheEventData10SizeType32NSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEvent7eventIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEvent7eventIdE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEvent7eventIdE"></span><span id="tensorrt_llm::executor::KVCacheEvent::eventId__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent_1ac35ffc03619ba4c47d466b1c64a9e477"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eventId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent7eventIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The unique id of this event. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEvent4dataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEvent4dataE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEvent4dataE"></span><span id="tensorrt_llm::executor::KVCacheEvent::data__KVCacheEventData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent_1a89f7a015393de6fd4b3617667debedcc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE" title="tensorrt_llm::executor::KVCacheEventData"><span class="n"><span class="pre">KVCacheEventData</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">data</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent4dataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The data corresponding to this event. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEvent10windowSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEvent10windowSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEvent10windowSizeE"></span><span id="tensorrt_llm::executor::KVCacheEvent::windowSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent_1acbefb82d64f1217ddfe1f4f79062eb51"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">windowSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent10windowSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The sliding window size. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12KVCacheEvent15attentionDpRankE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12KVCacheEvent15attentionDpRankE"></span><span id="_CPPv2N12tensorrt_llm8executor12KVCacheEvent15attentionDpRankE"></span><span id="tensorrt_llm::executor::KVCacheEvent::attentionDpRank__std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEvent_1a850554bdd55d03c5116c2db63c4e965e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">attentionDpRank</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent15attentionDpRankE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The attention DP rank of the event, if applicable. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE">
|
||
<span id="_CPPv3I0EN12tensorrt_llm8executor16KVCacheEventDiffE"></span><span id="_CPPv2I0EN12tensorrt_llm8executor16KVCacheEventDiffE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre"><</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">></span></span><br /><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEventDiff"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEventDiff</span></span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8oldValueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16KVCacheEventDiff8oldValueE"></span><span id="_CPPv2N12tensorrt_llm8executor16KVCacheEventDiff8oldValueE"></span><span id="tensorrt_llm::executor::KVCacheEventDiff::oldValue__T"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEventDiff_1a5eace570b46a93e06db3df99479eece4"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">oldValue</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8oldValueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8newValueE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor16KVCacheEventDiff8newValueE"></span><span id="_CPPv2N12tensorrt_llm8executor16KVCacheEventDiff8newValueE"></span><span id="tensorrt_llm::executor::KVCacheEventDiff::newValue__T"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheEventDiff_1aaf2dcad5ce01791fc1ef4a38ef683ed8"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">newValue</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8newValueE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KVCacheEventManagerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KVCacheEventManagerE"></span><span id="_CPPv2N12tensorrt_llm8executor19KVCacheEventManagerE"></span><span id="tensorrt_llm::executor::KVCacheEventManager"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KVCacheEventManager"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEventManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManagerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Exposes a limited set of KV cache manager functionalities. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KVCacheEventManager19KVCacheEventManagerENSt10shared_ptrIN12tensorrt_llm13batch_manager16kv_cache_manager18BaseKVCacheManagerEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KVCacheEventManager19KVCacheEventManagerENSt10shared_ptrIN12tensorrt_llm13batch_manager16kv_cache_manager18BaseKVCacheManagerEEE"></span><span id="_CPPv2N12tensorrt_llm8executor19KVCacheEventManager19KVCacheEventManagerENSt10shared_ptrIN12tensorrt_llm13batch_manager16kv_cache_manager18BaseKVCacheManagerEEE"></span><span id="tensorrt_llm::executor::KVCacheEventManager::KVCacheEventManager__std::shared_ptr:tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KVCacheEventManager_1acddf2441f42e1f19c7a06ac42b4dac6b"></span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheEventManager</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="tensorrt_llm::batch_manager::kv_cache_manager"><span class="n"><span class="pre">kv_cache_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">BaseKVCacheManager</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvCacheManager</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager19KVCacheEventManagerENSt10shared_ptrIN12tensorrt_llm13batch_manager16kv_cache_manager18BaseKVCacheManagerEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KVCacheEventManager15getLatestEventsENSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KVCacheEventManager15getLatestEventsENSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor19KVCacheEventManager15getLatestEventsENSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::KVCacheEventManager::getLatestEvents__std::optional:std::chrono::milliseconds:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KVCacheEventManager_1a42654b341294b34003bea8ed42587946"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE" title="tensorrt_llm::executor::KVCacheEvent"><span class="n"><span class="pre">KVCacheEvent</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLatestEvents</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">timeout</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager15getLatestEventsENSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the latest KV Cache events. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>timeout</strong> – The maximum time to wait for new events. If nullopt, will only return when new events are available, or when the executor instance has shutdown. </p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor19KVCacheEventManager14kvCacheManagerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor19KVCacheEventManager14kvCacheManagerE"></span><span id="_CPPv2N12tensorrt_llm8executor19KVCacheEventManager14kvCacheManagerE"></span><span id="tensorrt_llm::executor::KVCacheEventManager::kvCacheManager__std::shared_ptr:tensorrt_llm::batch_manager::kv_cache_manager::BaseKVCacheManager:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KVCacheEventManager_1a0a63a4b88e9a375232403da0ed20c403"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE" title="tensorrt_llm::batch_manager::kv_cache_manager"><span class="n"><span class="pre">kv_cache_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">BaseKVCacheManager</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kvCacheManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager14kvCacheManagerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheRemovedDataE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheRemovedDataE"></span><span id="tensorrt_llm::executor::KVCacheRemovedData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheRemovedData"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheRemovedData</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheRemovedData11blockHashesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheRemovedData11blockHashesE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheRemovedData11blockHashesE"></span><span id="tensorrt_llm::executor::KVCacheRemovedData::blockHashes__std::vector:IdType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheRemovedData_1a33b74e147613ec8f12a3fcc441eb6df4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">blockHashes</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedData11blockHashesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The hashes of blocks being removed. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfigE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for the request’s retention in the KV Cache. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigEv"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::KvCacheRetentionConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a1a64c8305f3a0ff4d2f7f981baa06c00"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigERKNSt6vectorI25TokenRangeRetentionConfigEE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE19KvCacheTransferModeRKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigERKNSt6vectorI25TokenRangeRetentionConfigEE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE19KvCacheTransferModeRKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigERKNSt6vectorI25TokenRangeRetentionConfigEE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE19KvCacheTransferModeRKNSt6stringE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::KvCacheRetentionConfig__std::vector:TokenRangeRetentionConfig:CR.RetentionPriority.std::optional:std::chrono::milliseconds:.KvCacheTransferMode.ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a963e5e14ce2c7a2955579f1899c81cad"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">tokenRangeRetentionPriorities</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">decodeRetentionPriority</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE" title="tensorrt_llm::executor::KvCacheRetentionConfig::kDefaultRetentionPriority"><span class="n"><span class="pre">kDefaultRetentionPriority</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">decodeDurationMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE" title="tensorrt_llm::executor::KvCacheTransferMode"><span class="n"><span class="pre">KvCacheTransferMode</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">transferMode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE" title="tensorrt_llm::executor::KvCacheTransferMode"><span class="n"><span class="pre">KvCacheTransferMode</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode4DRAME" title="tensorrt_llm::executor::KvCacheTransferMode::DRAM"><span class="n"><span class="pre">DRAM</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">directory</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">""</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigERKNSt6vectorI25TokenRangeRetentionConfigEE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE19KvCacheTransferModeRKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig29getTokenRangeRetentionConfigsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig29getTokenRangeRetentionConfigsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig29getTokenRangeRetentionConfigsEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getTokenRangeRetentionConfigsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a4f7eb9e150abff8c30521b3d2b11f68a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTokenRangeRetentionConfigs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig29getTokenRangeRetentionConfigsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig26getDecodeRetentionPriorityEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig26getDecodeRetentionPriorityEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig26getDecodeRetentionPriorityEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getDecodeRetentionPriorityC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a1be6f95e4aa00372070790c06a606f93"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDecodeRetentionPriority</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig26getDecodeRetentionPriorityEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig19getDecodeDurationMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig19getDecodeDurationMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig19getDecodeDurationMsEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getDecodeDurationMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1aee9c4b1af0e4f7650565ee3074b5f397"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDecodeDurationMs</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig19getDecodeDurationMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig15getTransferModeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig15getTransferModeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig15getTransferModeEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getTransferModeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1aa5c8153595583b1ab1953281321c3c4a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE" title="tensorrt_llm::executor::KvCacheTransferMode"><span class="n"><span class="pre">KvCacheTransferMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTransferMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig15getTransferModeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig12getDirectoryEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig12getDirectoryEv"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig12getDirectoryEv"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getDirectoryC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a000aa4b1729339bb539bfb0152a07b13"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDirectory</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig12getDirectoryEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig36getPerBlockRetentionPriorityDurationE10SizeType3210SizeType32">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig36getPerBlockRetentionPriorityDurationE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig36getPerBlockRetentionPriorityDurationE10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::getPerBlockRetentionPriorityDuration__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1afd4e5ccd65e62f5d6bdba3bef0b116f0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDurationE" title="tensorrt_llm::executor::RetentionPriorityAndDuration"><span class="n"><span class="pre">RetentionPriorityAndDuration</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPerBlockRetentionPriorityDuration</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">seqLen</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig36getPerBlockRetentionPriorityDurationE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Convert the token range data into an entry per kv block. Returns a tuple of vectors corresponding to the priorities and durations for each block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfigeqERK22KvCacheRetentionConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfigeqERK22KvCacheRetentionConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfigeqERK22KvCacheRetentionConfig"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::eq-operator__KvCacheRetentionConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a09bbe532da93bc4f10f06b76f5717d82"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfigeqERK22KvCacheRetentionConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMinRetentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig21kMinRetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig21kMinRetentionPriorityE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::kMinRetentionPriority__RetentionPriority"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1ad1775f789f5c8f01dc1546b65bc15750"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMinRetentionPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMinRetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMaxRetentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig21kMaxRetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig21kMaxRetentionPriorityE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::kMaxRetentionPriority__RetentionPriority"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1aa5bd07c90434c99284fec59df5590cb6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMaxRetentionPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">100</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMaxRetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::kDefaultRetentionPriority__RetentionPriority"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a6f575882c968d713624a82ff748b9da8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultRetentionPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">35</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig27mTokenRangeRetentionConfigsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig27mTokenRangeRetentionConfigsE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig27mTokenRangeRetentionConfigsE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::mTokenRangeRetentionConfigs__std::vector:TokenRangeRetentionConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1aaae6311e465a6fad29fd4cd7d1cd33f0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTokenRangeRetentionConfigs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig27mTokenRangeRetentionConfigsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The token ranges and priority levels to update. Ranges must be non-overlapping. For example [(0, 64), (100, 128), (70, 80)] is valid, whereas [(0, 64), (60, 128)] is not. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig24mDecodeRetentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig24mDecodeRetentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig24mDecodeRetentionPriorityE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::mDecodeRetentionPriority__RetentionPriority"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a6b1d57ec2bae317c057dde80f5b9cfdd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodeRetentionPriority</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig24mDecodeRetentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The priority level to assign to blocks allocated in the decode phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig17mDecodeDurationMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig17mDecodeDurationMsE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig17mDecodeDurationMsE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::mDecodeDurationMs__std::optional:std::chrono::milliseconds:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1aeff5b4895c608067ea663350021094fe"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodeDurationMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig17mDecodeDurationMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The duration in ms that decode blocks should remain at their assigned priority level. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig13mTransferModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig13mTransferModeE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig13mTransferModeE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::mTransferMode__KvCacheTransferMode"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a5ca662666b3272cd793e25712b36c3cf"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE" title="tensorrt_llm::executor::KvCacheTransferMode"><span class="n"><span class="pre">KvCacheTransferMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTransferMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig13mTransferModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The transfer mode for the block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig10mDirectoryE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig10mDirectoryE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig10mDirectoryE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::mDirectory__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1a2f437bae7aa4a6c77532a6962cb36791"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDirectory</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig10mDirectoryE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Name of the directory if transfer mode is GDS or POSIX_DEBUG_FALLBACK. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>A single entry to set block priorities over a token range. Earlier ranges always take priority over later ones. For example, with a block size of 16, a range of [0, 17] would be applied to the first two blocks. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig25TokenRangeRetentionConfigE10SizeType32NSt8optionalI10SizeType32EE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig25TokenRangeRetentionConfigE10SizeType32NSt8optionalI10SizeType32EE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig25TokenRangeRetentionConfigE10SizeType32NSt8optionalI10SizeType32EE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::TokenRangeRetentionConfig__SizeType32.std::optional:SizeType32:.RetentionPriority.std::optional:std::chrono::milliseconds:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1a01aa276ddfd1389b808557c6d9be6f73"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokenStart</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tokenEnd</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE" title="tensorrt_llm::executor::KvCacheRetentionConfig::kDefaultRetentionPriority"><span class="n"><span class="pre">kDefaultRetentionPriority</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">durationMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig25TokenRangeRetentionConfigE10SizeType32NSt8optionalI10SizeType32EE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigeqERK25TokenRangeRetentionConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigeqERK25TokenRangeRetentionConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigeqERK25TokenRangeRetentionConfig"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::eq-operator__TokenRangeRetentionConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1a58c0f14a585320a450319a21b8037824"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig"><span class="n"><span class="pre">TokenRangeRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigeqERK25TokenRangeRetentionConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10tokenStartE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10tokenStartE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10tokenStartE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::tokenStart__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1abdb050fcdd04f64406f1d72491fdeb1a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokenStart</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10tokenStartE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The first token of this range. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8tokenEndE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8tokenEndE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8tokenEndE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::tokenEnd__std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1a1665679cac7313b073d4906729a255c3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokenEnd</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8tokenEndE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The final token of this range. The end is not included in the range. This can be set to std::nullopt to extend the range to the end of the sequence. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8priorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8priorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8priorityE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::priority__RetentionPriority"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1a72224537a247f5003076f86892833696"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">priority</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8priorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The priority of this token range. Higher priorities are less likely to be evicted or offloaded. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10durationMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10durationMsE"></span><span id="_CPPv2N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10durationMsE"></span><span id="tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig::durationMs__std::optional:std::chrono::milliseconds:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KvCacheRetentionConfig_1_1TokenRangeRetentionConfig_1af8953ca7acc4db009d738bc10f70c2ab"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">durationMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10durationMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The duration in ms that the block should remain at the given priority level. Set to std::nullopt to have no expiration time, and keep the block at the given priority level until it gets reclaimed. After the duration has passed, the block will be moved back to the <code class="docutils literal notranslate"><span class="pre">kDefaultRetentionPriority</span></code> level. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockDataE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockDataE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>An entry for a single block stored into the tree. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::KVCacheStoredBlockData__IdType.tensorrt_llm::runtime::VecUniqueTokens.std::optional:tensorrt_llm::runtime::LoraTaskIdType:.SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1af6cc9927cdb952318da4d2eb2cf6eb31"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockHash</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime15VecUniqueTokensE" title="tensorrt_llm::runtime::VecUniqueTokens"><span class="n"><span class="pre">VecUniqueTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE" title="tensorrt_llm::runtime::LoraTaskIdType"><span class="n"><span class="pre">LoraTaskIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">loraId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cacheLevel</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::blockHash__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1a356cdfeb2a70ae9bd8d4be5c38fc1044"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">blockHash</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The hash of the block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::tokens__tensorrt_llm::runtime::VecUniqueTokens"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1a72b14db5340f183fae50810461400907"></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime15VecUniqueTokensE" title="tensorrt_llm::runtime::VecUniqueTokens"><span class="n"><span class="pre">VecUniqueTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The unique tokens of the block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::loraId__std::optional:tensorrt_llm::runtime::LoraTaskIdType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1adfeb0d2023e690a0d13079e7b7693d3d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE" title="tensorrt_llm::runtime::LoraTaskIdType"><span class="n"><span class="pre">LoraTaskIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loraId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The Lora task id of the block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::cacheLevel__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1aae5cef5af5ba1a253e9df4b26174629a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheLevel</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The cache level of the block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE"></span><span id="_CPPv2N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE"></span><span id="tensorrt_llm::executor::KVCacheStoredBlockData::priority__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredBlockData_1a8e2093038716a967af544a3c5e319629"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">priority</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The priority of the block. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17KVCacheStoredDataE"></span><span id="_CPPv2N12tensorrt_llm8executor17KVCacheStoredDataE"></span><span id="tensorrt_llm::executor::KVCacheStoredData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredData"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheStoredData</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17KVCacheStoredData10parentHashE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17KVCacheStoredData10parentHashE"></span><span id="_CPPv2N12tensorrt_llm8executor17KVCacheStoredData10parentHashE"></span><span id="tensorrt_llm::executor::KVCacheStoredData::parentHash__std::optional:IdType:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredData_1af93fe9da1a650d09d831591da6dbe913"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">parentHash</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredData10parentHashE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The parent of this sequence of stored blocks. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor17KVCacheStoredData6blocksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor17KVCacheStoredData6blocksE"></span><span id="_CPPv2N12tensorrt_llm8executor17KVCacheStoredData6blocksE"></span><span id="tensorrt_llm::executor::KVCacheStoredData::blocks__std::vector:KVCacheStoredBlockData:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheStoredData_1a02f84a6a706d9baece7dbd0f104cdff4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE" title="tensorrt_llm::executor::KVCacheStoredBlockData"><span class="n"><span class="pre">KVCacheStoredBlockData</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">blocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredData6blocksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>A sequence of blocks. The parent of block <code class="docutils literal notranslate"><span class="pre">i</span></code> is block <code class="docutils literal notranslate"><span class="pre">i-1</span></code></p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedDataE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedDataE"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdType"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdType"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::KVCacheUpdatedData__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1a5ac325874ac9863855823a798a88037c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockHash</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdTypeNSt8optionalI16KVCacheEventDiffI10SizeType32EEENSt8optionalI16KVCacheEventDiffI10SizeType32EEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdTypeNSt8optionalI16KVCacheEventDiffI10SizeType32EEENSt8optionalI16KVCacheEventDiffI10SizeType32EEE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdTypeNSt8optionalI16KVCacheEventDiffI10SizeType32EEENSt8optionalI16KVCacheEventDiffI10SizeType32EEE"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::KVCacheUpdatedData__IdType.std::optional:KVCacheEventDiff:SizeType32::.std::optional:KVCacheEventDiff:SizeType32::"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1ad172027ecc7d73a4cf7b35189a2aab92"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockHash</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">cacheLevel</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdTypeNSt8optionalI16KVCacheEventDiffI10SizeType32EEENSt8optionalI16KVCacheEventDiffI10SizeType32EEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData17cacheLevelUpdatedE10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData17cacheLevelUpdatedE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData17cacheLevelUpdatedE10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::cacheLevelUpdated__SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1ad7fe06e5893b1ecf0fe4032b52855164"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">cacheLevelUpdated</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">oldValue</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">newValue</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData17cacheLevelUpdatedE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData15priorityUpdatedE10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData15priorityUpdatedE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData15priorityUpdatedE10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::priorityUpdated__SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1a62feae8ded6c060b17f5c54cb41a58e0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE" title="tensorrt_llm::executor::KVCacheUpdatedData"><span class="n"><span class="pre">KVCacheUpdatedData</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">priorityUpdated</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">oldValue</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">newValue</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData15priorityUpdatedE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData9blockHashE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData9blockHashE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData9blockHashE"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::blockHash__IdType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1a5dc8d421acb9f870ee86307982f6c76b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">blockHash</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData9blockHashE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The hash of the updated block. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData10cacheLevelE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData10cacheLevelE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData10cacheLevelE"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::cacheLevel__std::optional:KVCacheEventDiff:SizeType32::"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1a9c9779aca96e1a3e06994ad690c17f9a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheLevel</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData10cacheLevelE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The updated value of the cacheLevel field. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData8priorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18KVCacheUpdatedData8priorityE"></span><span id="_CPPv2N12tensorrt_llm8executor18KVCacheUpdatedData8priorityE"></span><span id="tensorrt_llm::executor::KVCacheUpdatedData::priority__std::optional:KVCacheEventDiff:SizeType32::"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1KVCacheUpdatedData_1a1ef3af844442752610874e4bddf88c06"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE" title="tensorrt_llm::executor::KVCacheEventDiff"><span class="n"><span class="pre">KVCacheEventDiff</span></span></a><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">priority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData8priorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The updated value of the priority field. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfigE"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig25LogitsPostProcessorConfigENSt8optionalI22LogitsPostProcessorMapEENSt8optionalI26LogitsPostProcessorBatchedEEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig25LogitsPostProcessorConfigENSt8optionalI22LogitsPostProcessorMapEENSt8optionalI26LogitsPostProcessorBatchedEEb"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig25LogitsPostProcessorConfigENSt8optionalI22LogitsPostProcessorMapEENSt8optionalI26LogitsPostProcessorBatchedEEb"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::LogitsPostProcessorConfig__std::optional:LogitsPostProcessorMap:.std::optional:LogitsPostProcessorBatched:.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a296cabed99e1093aa70de87520278d8d"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LogitsPostProcessorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE" title="tensorrt_llm::executor::LogitsPostProcessorMap"><span class="n"><span class="pre">LogitsPostProcessorMap</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">processorMap</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE" title="tensorrt_llm::executor::LogitsPostProcessorBatched"><span class="n"><span class="pre">LogitsPostProcessorBatched</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">processorBatched</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">replicate</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig25LogitsPostProcessorConfigENSt8optionalI22LogitsPostProcessorMapEENSt8optionalI26LogitsPostProcessorBatchedEEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig15getProcessorMapEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25LogitsPostProcessorConfig15getProcessorMapEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25LogitsPostProcessorConfig15getProcessorMapEv"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::getProcessorMapC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a58f16d87581a027960f7a9a7b35e62f0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE" title="tensorrt_llm::executor::LogitsPostProcessorMap"><span class="n"><span class="pre">LogitsPostProcessorMap</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getProcessorMap</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig15getProcessorMapEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig19getProcessorBatchedEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25LogitsPostProcessorConfig19getProcessorBatchedEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25LogitsPostProcessorConfig19getProcessorBatchedEv"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::getProcessorBatchedC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a83f9ad50e26859cc6433867a536bd95b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE" title="tensorrt_llm::executor::LogitsPostProcessorBatched"><span class="n"><span class="pre">LogitsPostProcessorBatched</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getProcessorBatched</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig19getProcessorBatchedEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig12getReplicateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25LogitsPostProcessorConfig12getReplicateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor25LogitsPostProcessorConfig12getReplicateEv"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::getReplicateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1ae468e1f83b44da9a16b5e50e3b826b0f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getReplicate</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig12getReplicateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig15setProcessorMapERK22LogitsPostProcessorMap">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig15setProcessorMapERK22LogitsPostProcessorMap"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig15setProcessorMapERK22LogitsPostProcessorMap"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::setProcessorMap__LogitsPostProcessorMapCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a16259a87bd2e42ae52bfe98f4f01ee6e"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setProcessorMap</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE" title="tensorrt_llm::executor::LogitsPostProcessorMap"><span class="n"><span class="pre">LogitsPostProcessorMap</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">processorMap</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig15setProcessorMapERK22LogitsPostProcessorMap" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig19setProcessorBatchedERK26LogitsPostProcessorBatched">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig19setProcessorBatchedERK26LogitsPostProcessorBatched"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig19setProcessorBatchedERK26LogitsPostProcessorBatched"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::setProcessorBatched__LogitsPostProcessorBatchedCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a39acb85e8c744c1657bc9e8d3b527c95"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setProcessorBatched</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE" title="tensorrt_llm::executor::LogitsPostProcessorBatched"><span class="n"><span class="pre">LogitsPostProcessorBatched</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">processorBatched</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig19setProcessorBatchedERK26LogitsPostProcessorBatched" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig12setReplicateEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig12setReplicateEb"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig12setReplicateEb"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::setReplicate__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a07a324ea77a570ed567c305fefc7c305"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setReplicate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">replicate</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig12setReplicateEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig13mProcessorMapE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig13mProcessorMapE"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig13mProcessorMapE"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::mProcessorMap__std::optional:LogitsPostProcessorMap:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a4123aa8b24b56aa7c9d3d45d9367dda1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE" title="tensorrt_llm::executor::LogitsPostProcessorMap"><span class="n"><span class="pre">LogitsPostProcessorMap</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mProcessorMap</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig13mProcessorMapE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>mapping from post processor names to non-batched post processors </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig17mProcessorBatchedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig17mProcessorBatchedE"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig17mProcessorBatchedE"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::mProcessorBatched__std::optional:LogitsPostProcessorBatched:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1a6fbb062ef3127599168b3ba7dc2d8089"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE" title="tensorrt_llm::executor::LogitsPostProcessorBatched"><span class="n"><span class="pre">LogitsPostProcessorBatched</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mProcessorBatched</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig17mProcessorBatchedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>single batched post processor </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig10mReplicateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25LogitsPostProcessorConfig10mReplicateE"></span><span id="_CPPv2N12tensorrt_llm8executor25LogitsPostProcessorConfig10mReplicateE"></span><span id="tensorrt_llm::executor::LogitsPostProcessorConfig::mReplicate__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LogitsPostProcessorConfig_1ac2be6c0331013174f435f6fb0db3adf5"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mReplicate</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig10mReplicateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>If set to true, logits post processor will run on all TP ranks in last PP rank. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfigE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for Look-Ahead speculative decoding. Allows to include window size, ngram size and verification set size. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigE10SizeType3210SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::LookaheadDecodingConfig__SizeType32.SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1abef9e49b2e2e6312cf4c280146d24c0a"></span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">ngramSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">verificationSetSize</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigEv"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::LookaheadDecodingConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a2affd5248374a476986c6aadc51eac4f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfigeqERK23LookaheadDecodingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfigeqERK23LookaheadDecodingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfigeqERK23LookaheadDecodingConfig"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::eq-operator__LookaheadDecodingConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a419b3ac66d9136151a2a0b7d526ad9b3"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfigeqERK23LookaheadDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig3getEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig3getEv"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig3getEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::getC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a1c0c7073a69264dc8a9391f019df8490"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">tuple</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">get</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig3getEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig13getWindowSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig13getWindowSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig13getWindowSizeEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::getWindowSizeC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a8aa7577b18ff38bf4bc9640acf9aae47"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getWindowSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig13getWindowSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig12getNgramSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig12getNgramSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig12getNgramSizeEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::getNgramSizeC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a5b20a75ca2a5e7f08259f0500330e58e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNgramSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig12getNgramSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig22getVerificationSetSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig22getVerificationSetSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig22getVerificationSetSizeEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::getVerificationSetSizeC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1adad2572691ad3764b01c48b8ba2885c0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getVerificationSetSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig22getVerificationSetSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig28calculateSpeculativeResourceEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig28calculateSpeculativeResourceEv"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig28calculateSpeculativeResourceEv"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::calculateSpeculativeResourceC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a703a176c65e6095b092b1b389d02cb9b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">tuple</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">calculateSpeculativeResource</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig28calculateSpeculativeResourceEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>return <maxDecodingTokens, maxPathLen, maxDraftTokens, maxDraftPathLen> </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig4isLEERK23LookaheadDecodingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor23LookaheadDecodingConfig4isLEERK23LookaheadDecodingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor23LookaheadDecodingConfig4isLEERK23LookaheadDecodingConfig"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::isLE__LookaheadDecodingConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a87a081925bb740bf34f8c3a0a65f4180"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLE</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">that</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig4isLEERK23LookaheadDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>return true when <code class="docutils literal notranslate"><span class="pre">this</span></code> can be executed on resources defined by <code class="docutils literal notranslate"><span class="pre">that</span></code></p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig33calculateSpeculativeResourceTupleE10SizeType3210SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig33calculateSpeculativeResourceTupleE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig33calculateSpeculativeResourceTupleE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::calculateSpeculativeResourceTuple__SizeType32.SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a77f8ff2f179f05aea08e53653a772d02"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">tuple</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">calculateSpeculativeResourceTuple</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">ngramSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">verificationSetSize</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig33calculateSpeculativeResourceTupleE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig7isLegalE10SizeType3210SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig7isLegalE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig7isLegalE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::isLegal__SizeType32.SizeType32.SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1aec6d6b6c329339fe92c04b3c99606c27"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLegal</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">windowSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">ngramSize</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">verificationSetSize</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig7isLegalE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>return true when the parameter combination is valid. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig31kDefaultLookaheadDecodingWindowE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig31kDefaultLookaheadDecodingWindowE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig31kDefaultLookaheadDecodingWindowE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::kDefaultLookaheadDecodingWindow__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1af7b644f331cbd81d11ecc77227a6b7c0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultLookaheadDecodingWindow</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig31kDefaultLookaheadDecodingWindowE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig30kDefaultLookaheadDecodingNgramE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig30kDefaultLookaheadDecodingNgramE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig30kDefaultLookaheadDecodingNgramE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::kDefaultLookaheadDecodingNgram__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1af953d4f14b1ea95a62420e3491f73568"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultLookaheadDecodingNgram</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig30kDefaultLookaheadDecodingNgramE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig40kDefaultLookaheadDecodingVerificationSetE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig40kDefaultLookaheadDecodingVerificationSetE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig40kDefaultLookaheadDecodingVerificationSetE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::kDefaultLookaheadDecodingVerificationSet__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1adf860e7d6057a71426b9276daa9d8486"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultLookaheadDecodingVerificationSet</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig40kDefaultLookaheadDecodingVerificationSetE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig11mWindowSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig11mWindowSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig11mWindowSizeE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::mWindowSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a5325178fb1f33a47552ac21f4315e6fa"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mWindowSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig11mWindowSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig10mNgramSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig10mNgramSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig10mNgramSizeE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::mNgramSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a47e578a6e17991ce817a1763cf610e57"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNgramSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig10mNgramSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig20mVerificationSetSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor23LookaheadDecodingConfig20mVerificationSetSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor23LookaheadDecodingConfig20mVerificationSetSizeE"></span><span id="tensorrt_llm::executor::LookaheadDecodingConfig::mVerificationSetSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1LookaheadDecodingConfig_1a01deb81773d9c4ccf37074d773f5ca34"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mVerificationSetSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig20mVerificationSetSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10LoraConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10LoraConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor10LoraConfigE"></span><span id="tensorrt_llm::executor::LoraConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for LoRA. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10LoraConfig10LoraConfigE6IdTypeNSt8optionalI6TensorEENSt8optionalI6TensorEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10LoraConfig10LoraConfigE6IdTypeNSt8optionalI6TensorEENSt8optionalI6TensorEE"></span><span id="_CPPv2N12tensorrt_llm8executor10LoraConfig10LoraConfigE6IdTypeNSt8optionalI6TensorEENSt8optionalI6TensorEE"></span><span id="tensorrt_llm::executor::LoraConfig::LoraConfig__IdType.std::optional:Tensor:.std::optional:Tensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1a2a12dcb55bc38dd108c977c7b2fb4a55"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">weights</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig10LoraConfigE6IdTypeNSt8optionalI6TensorEENSt8optionalI6TensorEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor10LoraConfig9getTaskIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor10LoraConfig9getTaskIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor10LoraConfig9getTaskIdEv"></span><span id="tensorrt_llm::executor::LoraConfig::getTaskIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1ad1d8cbbfcbd2046bf1b73ce608582c1d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTaskId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig9getTaskIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor10LoraConfig10getWeightsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor10LoraConfig10getWeightsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor10LoraConfig10getWeightsEv"></span><span id="tensorrt_llm::executor::LoraConfig::getWeightsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1af3f46ffd3779842a130a0f658edcdd6f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getWeights</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig10getWeightsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor10LoraConfig9getConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor10LoraConfig9getConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor10LoraConfig9getConfigEv"></span><span id="tensorrt_llm::executor::LoraConfig::getConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1a17cb6307358a40303326dddcaf2971d3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig9getConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10LoraConfig7mTaskIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10LoraConfig7mTaskIdE"></span><span id="_CPPv2N12tensorrt_llm8executor10LoraConfig7mTaskIdE"></span><span id="tensorrt_llm::executor::LoraConfig::mTaskId__IdType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1a863ffa00f268192c572d4e3f0737d535"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTaskId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig7mTaskIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The Lora task id. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10LoraConfig8mWeightsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10LoraConfig8mWeightsE"></span><span id="_CPPv2N12tensorrt_llm8executor10LoraConfig8mWeightsE"></span><span id="tensorrt_llm::executor::LoraConfig::mWeights__std::optional:Tensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1ae6941906a80853143186df62f10d8d95"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mWeights</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig8mWeightsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The Lora weights. See TRT-LLM documentation for expected shapes and types. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor10LoraConfig7mConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor10LoraConfig7mConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor10LoraConfig7mConfigE"></span><span id="tensorrt_llm::executor::LoraConfig::mConfig__std::optional:Tensor:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1LoraConfig_1a70553cf896bdbb10718070e7da355bbc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig7mConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The Lora configuration. See TRT-LLM documentation for detailed description of the config tensor. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11MropeConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11MropeConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor11MropeConfigE"></span><span id="tensorrt_llm::executor::MropeConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MropeConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for mrope. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11MropeConfig11MropeConfigE6Tensor10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11MropeConfig11MropeConfigE6Tensor10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor11MropeConfig11MropeConfigE6Tensor10SizeType32"></span><span id="tensorrt_llm::executor::MropeConfig::MropeConfig__Tensor.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig_1a163652c73e6c583fc583dbd3afb74645"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MropeConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mropeRoratySinCos</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mropePositionDeltas</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig11MropeConfigE6Tensor10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11MropeConfig20getMRopeRotaryCosSinEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11MropeConfig20getMRopeRotaryCosSinEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11MropeConfig20getMRopeRotaryCosSinEv"></span><span id="tensorrt_llm::executor::MropeConfig::getMRopeRotaryCosSinC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig_1ad45065d1fea1393992dc7364a2fa5d84"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMRopeRotaryCosSin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11MropeConfig20getMRopeRotaryCosSinEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor11MropeConfig22getMRopePositionDeltasEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor11MropeConfig22getMRopePositionDeltasEv"></span><span id="_CPPv2NK12tensorrt_llm8executor11MropeConfig22getMRopePositionDeltasEv"></span><span id="tensorrt_llm::executor::MropeConfig::getMRopePositionDeltasC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig_1a5cc9b41eac1cc0bee6ffdd940357f056"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMRopePositionDeltas</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor11MropeConfig22getMRopePositionDeltasEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11MropeConfig18mMRopeRotaryCosSinE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11MropeConfig18mMRopeRotaryCosSinE"></span><span id="_CPPv2N12tensorrt_llm8executor11MropeConfig18mMRopeRotaryCosSinE"></span><span id="tensorrt_llm::executor::MropeConfig::mMRopeRotaryCosSin__Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig_1a958694179acab9bc19371c82cb887df1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMRopeRotaryCosSin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig18mMRopeRotaryCosSinE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The mrope rotary sin and cos cache. Expected shape: [maxPositionEmbeddings*rotaryEmbeddingDim],Data type must float32. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor11MropeConfig20mMRopePositionDeltasE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor11MropeConfig20mMRopePositionDeltasE"></span><span id="_CPPv2N12tensorrt_llm8executor11MropeConfig20mMRopePositionDeltasE"></span><span id="tensorrt_llm::executor::MropeConfig::mMRopePositionDeltas__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MropeConfig_1a34c4fbc4d6a6c1ee7d1a407c0596546b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMRopePositionDeltas</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig20mMRopePositionDeltasE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The mrope position deltas. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15MultimodalInputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15MultimodalInputE"></span><span id="_CPPv2N12tensorrt_llm8executor15MultimodalInputE"></span><span id="tensorrt_llm::executor::MultimodalInput"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MultimodalInput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Multimodal input data class. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15MultimodalInput15MultimodalInputENSt6vectorINSt6vectorI10SizeType32EEEENSt6vectorI10SizeType32EENSt6vectorI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15MultimodalInput15MultimodalInputENSt6vectorINSt6vectorI10SizeType32EEEENSt6vectorI10SizeType32EENSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor15MultimodalInput15MultimodalInputENSt6vectorINSt6vectorI10SizeType32EEEENSt6vectorI10SizeType32EENSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::executor::MultimodalInput::MultimodalInput__std::vector:std::vector:SizeType32::.std::vector:SizeType32:.std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1ae2fdf1d8dd612dfe4193d692c175292a"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MultimodalInput</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multimodalHashes</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multimodalPositions</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multimodalLengths</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput15MultimodalInputENSt6vectorINSt6vectorI10SizeType32EEEENSt6vectorI10SizeType32EENSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15MultimodalInput19getMultimodalHashesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15MultimodalInput19getMultimodalHashesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15MultimodalInput19getMultimodalHashesEv"></span><span id="tensorrt_llm::executor::MultimodalInput::getMultimodalHashesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1a69a1998b196e3234d40f73cb77f08dcb"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultimodalHashes</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput19getMultimodalHashesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15MultimodalInput22getMultimodalPositionsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15MultimodalInput22getMultimodalPositionsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15MultimodalInput22getMultimodalPositionsEv"></span><span id="tensorrt_llm::executor::MultimodalInput::getMultimodalPositionsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1affbf7a499a907a777819bba94a08e3a8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultimodalPositions</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput22getMultimodalPositionsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15MultimodalInput20getMultimodalLengthsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15MultimodalInput20getMultimodalLengthsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15MultimodalInput20getMultimodalLengthsEv"></span><span id="tensorrt_llm::executor::MultimodalInput::getMultimodalLengthsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1a966bc4554314a394e144e2f1f89d9349"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultimodalLengths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput20getMultimodalLengthsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15MultimodalInput17mMultimodalHashesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15MultimodalInput17mMultimodalHashesE"></span><span id="_CPPv2N12tensorrt_llm8executor15MultimodalInput17mMultimodalHashesE"></span><span id="tensorrt_llm::executor::MultimodalInput::mMultimodalHashes__std::vector:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1a00014fd00f0c80a12a74fd1fa5f9fe20"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMultimodalHashes</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput17mMultimodalHashesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The multimodal hashes. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15MultimodalInput20mMultimodalPositionsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15MultimodalInput20mMultimodalPositionsE"></span><span id="_CPPv2N12tensorrt_llm8executor15MultimodalInput20mMultimodalPositionsE"></span><span id="tensorrt_llm::executor::MultimodalInput::mMultimodalPositions__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1af8d670fddf552e181d5f2cbd8aedcb8d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMultimodalPositions</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput20mMultimodalPositionsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The multimodal positions. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15MultimodalInput18mMultimodalLengthsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15MultimodalInput18mMultimodalLengthsE"></span><span id="_CPPv2N12tensorrt_llm8executor15MultimodalInput18mMultimodalLengthsE"></span><span id="tensorrt_llm::executor::MultimodalInput::mMultimodalLengths__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1MultimodalInput_1ab19a53dd86441f78f7e9adb865038fea"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMultimodalLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput18mMultimodalLengthsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The multimodal lengths. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfigE"></span><span id="tensorrt_llm::executor::OrchestratorConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">OrchestratorConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig18OrchestratorConfigEbNSt6stringENSt10shared_ptrIN3mpi7MpiCommEEEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig18OrchestratorConfigEbNSt6stringENSt10shared_ptrIN3mpi7MpiCommEEEb"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig18OrchestratorConfigEbNSt6stringENSt10shared_ptrIN3mpi7MpiCommEEEb"></span><span id="tensorrt_llm::executor::OrchestratorConfig::OrchestratorConfig__b.ss.std::shared_ptr:mpi::MpiComm:.b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a48c20f133f76c052ad8d9174462648f4"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">OrchestratorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isOrchestrator</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">workerExecutablePath</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">""</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm3mpiE" title="tensorrt_llm::mpi"><span class="n"><span class="pre">mpi</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">MpiComm</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">orchLeaderComm</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">spawnProcesses</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig18OrchestratorConfigEbNSt6stringENSt10shared_ptrIN3mpi7MpiCommEEEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getIsOrchestratorEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18OrchestratorConfig17getIsOrchestratorEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18OrchestratorConfig17getIsOrchestratorEv"></span><span id="tensorrt_llm::executor::OrchestratorConfig::getIsOrchestratorC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a0faa15a0f3ce88cb25eb07a1e1a91d34"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIsOrchestrator</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getIsOrchestratorEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig23getWorkerExecutablePathEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18OrchestratorConfig23getWorkerExecutablePathEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18OrchestratorConfig23getWorkerExecutablePathEv"></span><span id="tensorrt_llm::executor::OrchestratorConfig::getWorkerExecutablePathC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a603abfa699a11b7e910278cc9c027542"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getWorkerExecutablePath</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig23getWorkerExecutablePathEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getOrchLeaderCommEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18OrchestratorConfig17getOrchLeaderCommEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18OrchestratorConfig17getOrchLeaderCommEv"></span><span id="tensorrt_llm::executor::OrchestratorConfig::getOrchLeaderCommC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1adeb40968afe900fff60a0ee58e500cff"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm3mpiE" title="tensorrt_llm::mpi"><span class="n"><span class="pre">mpi</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">MpiComm</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOrchLeaderComm</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getOrchLeaderCommEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getSpawnProcessesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18OrchestratorConfig17getSpawnProcessesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18OrchestratorConfig17getSpawnProcessesEv"></span><span id="tensorrt_llm::executor::OrchestratorConfig::getSpawnProcessesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a1b3409f1355ea612ac0c1fdae0c05ded"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpawnProcesses</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getSpawnProcessesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setIsOrchestratorEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig17setIsOrchestratorEb"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig17setIsOrchestratorEb"></span><span id="tensorrt_llm::executor::OrchestratorConfig::setIsOrchestrator__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a5384e155759356518a2bca388a1ae1d0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setIsOrchestrator</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isOrchestrator</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setIsOrchestratorEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig23setWorkerExecutablePathERKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig23setWorkerExecutablePathERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig23setWorkerExecutablePathERKNSt6stringE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::setWorkerExecutablePath__ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a557913170113b0f37daab422bb8552d1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setWorkerExecutablePath</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">workerExecutablePath</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig23setWorkerExecutablePathERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setOrchLeaderCommERKNSt10shared_ptrIN3mpi7MpiCommEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig17setOrchLeaderCommERKNSt10shared_ptrIN3mpi7MpiCommEEE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig17setOrchLeaderCommERKNSt10shared_ptrIN3mpi7MpiCommEEE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::setOrchLeaderComm__std::shared_ptr:mpi::MpiComm:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1ab4e88541c7a38d3675485f227030f7e2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setOrchLeaderComm</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm3mpiE" title="tensorrt_llm::mpi"><span class="n"><span class="pre">mpi</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">MpiComm</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">orchLeaderComm</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setOrchLeaderCommERKNSt10shared_ptrIN3mpi7MpiCommEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setSpawnProcessesEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig17setSpawnProcessesEb"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig17setSpawnProcessesEb"></span><span id="tensorrt_llm::executor::OrchestratorConfig::setSpawnProcesses__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a32a045e475fb0515f1959022b1d56ba1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSpawnProcesses</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">spawnProcesses</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setSpawnProcessesEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mIsOrchestratorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig15mIsOrchestratorE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig15mIsOrchestratorE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::mIsOrchestrator__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a272f1d1692fae68ed8242d8180326977"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIsOrchestrator</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mIsOrchestratorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig21mWorkerExecutablePathE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig21mWorkerExecutablePathE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig21mWorkerExecutablePathE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::mWorkerExecutablePath__ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a8a4c20267c20632d8107e9b8041198c2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mWorkerExecutablePath</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig21mWorkerExecutablePathE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mOrchLeaderCommE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig15mOrchLeaderCommE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig15mOrchLeaderCommE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::mOrchLeaderComm__std::shared_ptr:mpi::MpiComm:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1a16ac5a458ad5e4e048d0badcba4b54d5"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm3mpiE" title="tensorrt_llm::mpi"><span class="n"><span class="pre">mpi</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">MpiComm</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOrchLeaderComm</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mOrchLeaderCommE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mSpawnProcessesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18OrchestratorConfig15mSpawnProcessesE"></span><span id="_CPPv2N12tensorrt_llm8executor18OrchestratorConfig15mSpawnProcessesE"></span><span id="tensorrt_llm::executor::OrchestratorConfig::mSpawnProcesses__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OrchestratorConfig_1af4f62bc30dd01bfd0ac44b7dfd62323d"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSpawnProcesses</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mSpawnProcessesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfigE"></span><span id="tensorrt_llm::executor::OutputConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">OutputConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration that controls the outputs of a <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a>. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig12OutputConfigEbbbbbbNSt8optionalINSt6vectorI21AdditionalModelOutputEEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig12OutputConfigEbbbbbbNSt8optionalINSt6vectorI21AdditionalModelOutputEEEE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig12OutputConfigEbbbbbbNSt8optionalINSt6vectorI21AdditionalModelOutputEEEE"></span><span id="tensorrt_llm::executor::OutputConfig::OutputConfig__b.b.b.b.b.b.std::optional:std::vector:AdditionalModelOutput::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a8c12a4cfdd13458d3b2d3817ccd42e65"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">OutputConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnLogProbs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnContextLogits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnGenerationLogits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">excludeInputFromOutput</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnEncoderOutput</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnPerfMetrics</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">additionalModelOutputs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig12OutputConfigEbbbbbbNSt8optionalINSt6vectorI21AdditionalModelOutputEEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig14returnLogProbsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig14returnLogProbsE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig14returnLogProbsE"></span><span id="tensorrt_llm::executor::OutputConfig::returnLogProbs__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a1f23a994b65629c8beee0bbb79f71ee7"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">returnLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig14returnLogProbsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should contain log probabilities. Default is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig19returnContextLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig19returnContextLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig19returnContextLogitsE"></span><span id="tensorrt_llm::executor::OutputConfig::returnContextLogits__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a4419de36cc2f99f8c4ce8911ee0dadf5"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">returnContextLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig19returnContextLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should contain the context logits. Default is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig22returnGenerationLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig22returnGenerationLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig22returnGenerationLogitsE"></span><span id="tensorrt_llm::executor::OutputConfig::returnGenerationLogits__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a1d8dda4081c539f18f32af0be70c0f17"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">returnGenerationLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22returnGenerationLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should contain the generation logits. Default is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig22excludeInputFromOutputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig22excludeInputFromOutputE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig22excludeInputFromOutputE"></span><span id="tensorrt_llm::executor::OutputConfig::excludeInputFromOutput__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1ab38293c37f6f6b60e3e39f68f18f7b90"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">excludeInputFromOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22excludeInputFromOutputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if output tokens in <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should include the input tokens. Default is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig19returnEncoderOutputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig19returnEncoderOutputE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig19returnEncoderOutputE"></span><span id="tensorrt_llm::executor::OutputConfig::returnEncoderOutput__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a11049bc5a1b255c703533515539ec66a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">returnEncoderOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig19returnEncoderOutputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should contain encoder output hidden states (for encoder-only and encoder-decoder models). Default is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig17returnPerfMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig17returnPerfMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig17returnPerfMetricsE"></span><span id="tensorrt_llm::executor::OutputConfig::returnPerfMetrics__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a7d6aef80893aa3eabbc78f0a504f8e1a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">returnPerfMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig17returnPerfMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls if <a class="reference internal" href="#structtensorrt__llm_1_1executor_1_1Result"><span class="std std-ref">Result</span></a> should contain performance metrics. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor12OutputConfig22additionalModelOutputsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor12OutputConfig22additionalModelOutputsE"></span><span id="_CPPv2N12tensorrt_llm8executor12OutputConfig22additionalModelOutputsE"></span><span id="tensorrt_llm::executor::OutputConfig::additionalModelOutputs__std::optional:std::vector:AdditionalModelOutput::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1OutputConfig_1a2bedd4909a23305c623bb6c424aea635"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE" title="tensorrt_llm::executor::AdditionalModelOutput"><span class="n"><span class="pre">AdditionalModelOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">additionalModelOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22additionalModelOutputsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The additional outputs to gather from the model. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfigE"></span><span id="tensorrt_llm::executor::ParallelConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ParallelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>A configuration class for the parallel execution parameters Currently only supports commType = CommunicationType::kMPI. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig14ParallelConfigE17CommunicationType17CommunicationModeNSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI18OrchestratorConfigEENSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig14ParallelConfigE17CommunicationType17CommunicationModeNSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI18OrchestratorConfigEENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig14ParallelConfigE17CommunicationType17CommunicationModeNSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI18OrchestratorConfigEENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::ParallelConfig::ParallelConfig__CommunicationType.CommunicationMode.std::optional:std::vector:SizeType32::.std::optional:std::vector:SizeType32::.std::optional:OrchestratorConfig:CR.std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a4d19e41f220b7b389c80ed0d88e38d18"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ParallelConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="tensorrt_llm::executor::CommunicationType"><span class="n"><span class="pre">CommunicationType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">commType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="tensorrt_llm::executor::CommunicationType"><span class="n"><span class="pre">CommunicationType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationType4kMPIE" title="tensorrt_llm::executor::CommunicationType::kMPI"><span class="n"><span class="pre">kMPI</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="tensorrt_llm::executor::CommunicationMode"><span class="n"><span class="pre">CommunicationMode</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">commMode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="tensorrt_llm::executor::CommunicationMode"><span class="n"><span class="pre">CommunicationMode</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationMode7kLEADERE" title="tensorrt_llm::executor::CommunicationMode::kLEADER"><span class="n"><span class="pre">kLEADER</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">deviceIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">participantIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">orchestratorConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">numNodes</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig14ParallelConfigE17CommunicationType17CommunicationModeNSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI18OrchestratorConfigEENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Constructor. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>commType</strong> – The communication type. See CommunicationType. </p></li>
|
||
<li><p><strong>commMode</strong> – The communication mode. See CommunicationMode. </p></li>
|
||
<li><p><strong>deviceIds</strong> – The IDs of the GPUs involved in the execution of the model </p></li>
|
||
<li><p><strong>participantIds</strong> – The participant IDs (MPI ranks if commType == kMPI) involved in the execution of the model. The first participant is considered to be the leader. </p></li>
|
||
<li><p><strong>orchestratorConfig</strong> – The orchestrator configuration. See <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1OrchestratorConfig"><span class="std std-ref">OrchestratorConfig</span></a>. </p></li>
|
||
<li><p><strong>numNodes</strong> – The number of nodes to use for execution. Default is 1. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig20getCommunicationTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig20getCommunicationTypeEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getCommunicationTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ad384d17a72f351d994e604594d6a6f4b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="tensorrt_llm::executor::CommunicationType"><span class="n"><span class="pre">CommunicationType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCommunicationType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationModeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig20getCommunicationModeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig20getCommunicationModeEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getCommunicationModeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a9bf12e33fc86542ee77ff1488fa7835c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="tensorrt_llm::executor::CommunicationMode"><span class="n"><span class="pre">CommunicationMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCommunicationMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationModeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig12getDeviceIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig12getDeviceIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig12getDeviceIdsEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getDeviceIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a4c0de3df9bae6b2261f9b65df7a07da9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDeviceIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig12getDeviceIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig17getParticipantIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig17getParticipantIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig17getParticipantIdsEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getParticipantIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a5c24e64fa629a2291c11959faa03cd3a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getParticipantIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig17getParticipantIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig21getOrchestratorConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig21getOrchestratorConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig21getOrchestratorConfigEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getOrchestratorConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a51b3d7fe63d40e0e99d06bd04df19aa9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOrchestratorConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig21getOrchestratorConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14ParallelConfig11getNumNodesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14ParallelConfig11getNumNodesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14ParallelConfig11getNumNodesEv"></span><span id="tensorrt_llm::executor::ParallelConfig::getNumNodesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a482c523ed1ec9bdc116db127adc8be64"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumNodes</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig11getNumNodesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationTypeE17CommunicationType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig20setCommunicationTypeE17CommunicationType"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig20setCommunicationTypeE17CommunicationType"></span><span id="tensorrt_llm::executor::ParallelConfig::setCommunicationType__CommunicationType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ad16a96f04fa3474bf67452910f64a8bc"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCommunicationType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="tensorrt_llm::executor::CommunicationType"><span class="n"><span class="pre">CommunicationType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationTypeE17CommunicationType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationModeE17CommunicationMode">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig20setCommunicationModeE17CommunicationMode"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig20setCommunicationModeE17CommunicationMode"></span><span id="tensorrt_llm::executor::ParallelConfig::setCommunicationMode__CommunicationMode"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a1bb2819d5c1add8908d9db259b16c741"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCommunicationMode</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="tensorrt_llm::executor::CommunicationMode"><span class="n"><span class="pre">CommunicationMode</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mode</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationModeE17CommunicationMode" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig12setDeviceIdsERKNSt6vectorI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig12setDeviceIdsERKNSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig12setDeviceIdsERKNSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::executor::ParallelConfig::setDeviceIds__std::vector:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a6bcd6e1acdde379946d599e884fb7ee1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDeviceIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">deviceIds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig12setDeviceIdsERKNSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig17setParticipantIdsERKNSt6vectorI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig17setParticipantIdsERKNSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig17setParticipantIdsERKNSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::executor::ParallelConfig::setParticipantIds__std::vector:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a7e3f6c11ef029cdee705305ca1129a0a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setParticipantIds</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">participantIds</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig17setParticipantIdsERKNSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig21setOrchestratorConfigERK18OrchestratorConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig21setOrchestratorConfigERK18OrchestratorConfig"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig21setOrchestratorConfigERK18OrchestratorConfig"></span><span id="tensorrt_llm::executor::ParallelConfig::setOrchestratorConfig__OrchestratorConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a72057d9d07087fc7b6fdb2795bf1fb5e"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setOrchestratorConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">orchestratorConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig21setOrchestratorConfigERK18OrchestratorConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig11setNumNodesE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig11setNumNodesE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig11setNumNodesE10SizeType32"></span><span id="tensorrt_llm::executor::ParallelConfig::setNumNodes__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ad12c45bb3b924b19bca0448b47f91202"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumNodes</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numNodes</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig11setNumNodesE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig9mCommTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig9mCommTypeE"></span><span id="tensorrt_llm::executor::ParallelConfig::mCommType__CommunicationType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ae1b162dc41f8541aba52d90c6bd76d18"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE" title="tensorrt_llm::executor::CommunicationType"><span class="n"><span class="pre">CommunicationType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCommType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The type of communication protocol used. Default is MPI. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommModeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig9mCommModeE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig9mCommModeE"></span><span id="tensorrt_llm::executor::ParallelConfig::mCommMode__CommunicationMode"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ad5b06cc50716576658e25aa7d30fd6b6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE" title="tensorrt_llm::executor::CommunicationMode"><span class="n"><span class="pre">CommunicationMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCommMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommModeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The mode of communication. See CommunicationMode. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig10mDeviceIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig10mDeviceIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig10mDeviceIdsE"></span><span id="tensorrt_llm::executor::ParallelConfig::mDeviceIds__std::optional:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a4a9207031d67fbe267d38d89c8901e81"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDeviceIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig10mDeviceIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The GPU device ids to use for executing this model. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig15mParticipantIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig15mParticipantIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig15mParticipantIdsE"></span><span id="tensorrt_llm::executor::ParallelConfig::mParticipantIds__std::optional:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1ae0e3d3936b187b3fed26b16220ee8a57"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mParticipantIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig15mParticipantIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The participant ids (MPI ranks for example) used for executing this model. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig19mOrchestratorConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig19mOrchestratorConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig19mOrchestratorConfigE"></span><span id="tensorrt_llm::executor::ParallelConfig::mOrchestratorConfig__std::optional:OrchestratorConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1a0a8ccedf7de365ca80be267e90627954"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE" title="tensorrt_llm::executor::OrchestratorConfig"><span class="n"><span class="pre">OrchestratorConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOrchestratorConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig19mOrchestratorConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Optional orchestrator configuration. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14ParallelConfig9mNumNodesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14ParallelConfig9mNumNodesE"></span><span id="_CPPv2N12tensorrt_llm8executor14ParallelConfig9mNumNodesE"></span><span id="tensorrt_llm::executor::ParallelConfig::mNumNodes__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1ParallelConfig_1aeddcc50a524103f0a0f8d11827d6cdb0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumNodes</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mNumNodesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The number of nodes to use for execution. Default is 1. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfigE"></span><span id="tensorrt_llm::executor::PeftCacheConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PeftCacheConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>config for PeftCacheManager </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15PeftCacheConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalIfEERKNSt8optionalI6size_tEERKNSt8optionalINSt6stringEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig15PeftCacheConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalIfEERKNSt8optionalI6size_tEERKNSt8optionalINSt6stringEEE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig15PeftCacheConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalIfEERKNSt8optionalI6size_tEERKNSt8optionalINSt6stringEEE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::PeftCacheConfig__SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.std::optional:float:CR.std::optional:s:CR.std::optional:ss:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a69175537bdffff566bffe50318b6f338"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PeftCacheConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numHostModuleLayer</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numDeviceModuleLayer</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">optimalAdapterSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE" title="tensorrt_llm::executor::PeftCacheConfig::kDefaultOptimalAdapterSize"><span class="n"><span class="pre">kDefaultOptimalAdapterSize</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAdapterSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE" title="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxAdapterSize"><span class="n"><span class="pre">kDefaultMaxAdapterSize</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numPutWorkers</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numEnsureWorkers</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCopyStreams</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxPagesPerBlockHost</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE" title="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxPagesPerBlockHost"><span class="n"><span class="pre">kDefaultMaxPagesPerBlockHost</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxPagesPerBlockDevice</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE" title="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxPagesPerBlockDevice"><span class="n"><span class="pre">kDefaultMaxPagesPerBlockDevice</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">deviceCachePercent</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">hostCacheSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">loraPrefetchDir</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15PeftCacheConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalIfEERKNSt8optionalI6size_tEERKNSt8optionalINSt6stringEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfigeqERK15PeftCacheConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfigeqERK15PeftCacheConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfigeqERK15PeftCacheConfig"></span><span id="tensorrt_llm::executor::PeftCacheConfig::eq-operator__PeftCacheConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ab60292786cb2f414b52a8d571708a281"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE" title="tensorrt_llm::executor::PeftCacheConfig"><span class="n"><span class="pre">PeftCacheConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfigeqERK15PeftCacheConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getNumHostModuleLayerEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig21getNumHostModuleLayerEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig21getNumHostModuleLayerEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getNumHostModuleLayerC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ad0a03ff1b983f8c306912156a370d436"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumHostModuleLayer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getNumHostModuleLayerEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getNumDeviceModuleLayerEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig23getNumDeviceModuleLayerEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig23getNumDeviceModuleLayerEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getNumDeviceModuleLayerC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ac6585fa80da031f50cfd241dab5516d4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumDeviceModuleLayer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getNumDeviceModuleLayerEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getOptimalAdapterSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig21getOptimalAdapterSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig21getOptimalAdapterSizeEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getOptimalAdapterSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a550e64682974fb60078a5c858d84bcc2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOptimalAdapterSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getOptimalAdapterSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getMaxAdapterSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig17getMaxAdapterSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig17getMaxAdapterSizeEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getMaxAdapterSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a7e11e64fc78e41f098df0a9b2d683939"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxAdapterSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getMaxAdapterSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getNumPutWorkersEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig16getNumPutWorkersEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig16getNumPutWorkersEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getNumPutWorkersC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1af92e38b38cb61de1dad2210a76011163"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumPutWorkers</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getNumPutWorkersEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig19getNumEnsureWorkersEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig19getNumEnsureWorkersEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig19getNumEnsureWorkersEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getNumEnsureWorkersC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1abcaf1eefd4d9ef5fbffa47eb885b1b6d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumEnsureWorkers</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig19getNumEnsureWorkersEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getNumCopyStreamsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig17getNumCopyStreamsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig17getNumCopyStreamsEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getNumCopyStreamsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a3e28ede570906ef9c3ccfc2898e0e4aa"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumCopyStreams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getNumCopyStreamsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getMaxPagesPerBlockHostEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig23getMaxPagesPerBlockHostEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig23getMaxPagesPerBlockHostEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getMaxPagesPerBlockHostC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1af7ab8d350fee59f9890e048393bf5a02"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPagesPerBlockHost</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getMaxPagesPerBlockHostEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig25getMaxPagesPerBlockDeviceEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig25getMaxPagesPerBlockDeviceEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig25getMaxPagesPerBlockDeviceEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getMaxPagesPerBlockDeviceC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1aa7147fa1d4bbef5ce8cc038b5434cc8e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPagesPerBlockDevice</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig25getMaxPagesPerBlockDeviceEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getDeviceCachePercentEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig21getDeviceCachePercentEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig21getDeviceCachePercentEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getDeviceCachePercentC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a5550bf7fe17f230b013e2e754633d80c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDeviceCachePercent</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getDeviceCachePercentEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getHostCacheSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig16getHostCacheSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig16getHostCacheSizeEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getHostCacheSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1abc33e84d2e2764a885fa326814c3a4c6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getHostCacheSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getHostCacheSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig18getLoraPrefetchDirEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15PeftCacheConfig18getLoraPrefetchDirEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15PeftCacheConfig18getLoraPrefetchDirEv"></span><span id="tensorrt_llm::executor::PeftCacheConfig::getLoraPrefetchDirC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ab585b856341ce309dd554787b271ab74"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLoraPrefetchDir</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig18getLoraPrefetchDirEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::kDefaultOptimalAdapterSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a669e2b2b2959e186fd9a23a1804f2373"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultOptimalAdapterSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxAdapterSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1aedca3e5c48f032a321e6e3edaadc2d5e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxAdapterSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">64</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxPagesPerBlockHost__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a85227290ca63d58758e17ef05368fff2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxPagesPerBlockHost</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">24</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::kDefaultMaxPagesPerBlockDevice__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a8f3b9a6868ff0b03e63e37d98e108839"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultMaxPagesPerBlockDevice</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mNumHostModuleLayerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig19mNumHostModuleLayerE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig19mNumHostModuleLayerE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mNumHostModuleLayer__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ae3a957e6c10ff32fda9cdfa261307a23"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumHostModuleLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mNumHostModuleLayerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mNumDeviceModuleLayerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig21mNumDeviceModuleLayerE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig21mNumDeviceModuleLayerE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mNumDeviceModuleLayer__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a6c734def4b320c40ec7758a2f9f2f928"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumDeviceModuleLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mNumDeviceModuleLayerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mOptimalAdapterSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig19mOptimalAdapterSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig19mOptimalAdapterSizeE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mOptimalAdapterSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a4cbcea2497e456f7f5c6988d9afba569"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOptimalAdapterSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mOptimalAdapterSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mMaxAdapterSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig15mMaxAdapterSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig15mMaxAdapterSizeE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mMaxAdapterSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1abe38d2b75a813f5b2e73e5800083984a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxAdapterSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mMaxAdapterSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mNumPutWorkersE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig14mNumPutWorkersE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig14mNumPutWorkersE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mNumPutWorkers__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1ab5b4bc29cc88c7cc3dd7f90bd8632966"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumPutWorkers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mNumPutWorkersE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig17mNumEnsureWorkersE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig17mNumEnsureWorkersE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig17mNumEnsureWorkersE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mNumEnsureWorkers__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a0e10d459eaf84adcd6d50926775f5a45"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumEnsureWorkers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig17mNumEnsureWorkersE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mNumCopyStreamsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig15mNumCopyStreamsE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig15mNumCopyStreamsE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mNumCopyStreams__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a2d7ccbe5d9c11d02cab32a1a7557c511"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumCopyStreams</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mNumCopyStreamsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mMaxPagesPerBlockHostE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig21mMaxPagesPerBlockHostE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig21mMaxPagesPerBlockHostE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mMaxPagesPerBlockHost__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a968719a3d4bec709899c4bbb50e84a6f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxPagesPerBlockHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mMaxPagesPerBlockHostE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig23mMaxPagesPerBlockDeviceE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig23mMaxPagesPerBlockDeviceE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig23mMaxPagesPerBlockDeviceE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mMaxPagesPerBlockDevice__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1abd1029f5855b119ea0b6c66c46e59eee"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxPagesPerBlockDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig23mMaxPagesPerBlockDeviceE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mDeviceCachePercentE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig19mDeviceCachePercentE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig19mDeviceCachePercentE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mDeviceCachePercent__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a838668e3174dd46dddb8bb60a9da0e05"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDeviceCachePercent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mDeviceCachePercentE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mHostCacheSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig14mHostCacheSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig14mHostCacheSizeE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mHostCacheSize__std::optional:s:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a656930138cb6f162e7fc15c82778d1cc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mHostCacheSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mHostCacheSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15PeftCacheConfig16mLoraPrefetchDirE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15PeftCacheConfig16mLoraPrefetchDirE"></span><span id="_CPPv2N12tensorrt_llm8executor15PeftCacheConfig16mLoraPrefetchDirE"></span><span id="tensorrt_llm::executor::PeftCacheConfig::mLoraPrefetchDir__std::optional:ss:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PeftCacheConfig_1a52f72b3e23ce6ea2b57a1fd86d5efd08"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLoraPrefetchDir</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig16mLoraPrefetchDirE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18PromptTuningConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor18PromptTuningConfigE"></span><span id="tensorrt_llm::executor::PromptTuningConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PromptTuningConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for prompt tuning. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18PromptTuningConfig18PromptTuningConfigE6TensorNSt8optionalI16VecTokenExtraIdsEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18PromptTuningConfig18PromptTuningConfigE6TensorNSt8optionalI16VecTokenExtraIdsEE"></span><span id="_CPPv2N12tensorrt_llm8executor18PromptTuningConfig18PromptTuningConfigE6TensorNSt8optionalI16VecTokenExtraIdsEE"></span><span id="tensorrt_llm::executor::PromptTuningConfig::PromptTuningConfig__Tensor.std::optional:VecTokenExtraIds:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig_1a5ef30446f49901fe54cf7d539be9a158"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PromptTuningConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">embeddingTable</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE" title="tensorrt_llm::executor::VecTokenExtraIds"><span class="n"><span class="pre">VecTokenExtraIds</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inputTokenExtraIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig18PromptTuningConfigE6TensorNSt8optionalI16VecTokenExtraIdsEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig17getEmbeddingTableEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18PromptTuningConfig17getEmbeddingTableEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18PromptTuningConfig17getEmbeddingTableEv"></span><span id="tensorrt_llm::executor::PromptTuningConfig::getEmbeddingTableC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig_1af8534fb4246a662291be6f7e17655ca0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEmbeddingTable</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig17getEmbeddingTableEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig21getInputTokenExtraIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor18PromptTuningConfig21getInputTokenExtraIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor18PromptTuningConfig21getInputTokenExtraIdsEv"></span><span id="tensorrt_llm::executor::PromptTuningConfig::getInputTokenExtraIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig_1ad7693f426c620eec280bf66202e22b34"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE" title="tensorrt_llm::executor::VecTokenExtraIds"><span class="n"><span class="pre">VecTokenExtraIds</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getInputTokenExtraIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig21getInputTokenExtraIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18PromptTuningConfig15mEmbeddingTableE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18PromptTuningConfig15mEmbeddingTableE"></span><span id="_CPPv2N12tensorrt_llm8executor18PromptTuningConfig15mEmbeddingTableE"></span><span id="tensorrt_llm::executor::PromptTuningConfig::mEmbeddingTable__Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig_1a4f6be4050b7b94a5950234e70b9df919"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEmbeddingTable</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig15mEmbeddingTableE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The prompt embedding table. Expected shape: [task vocab_size, hidden_size]. Data type must match model weights. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor18PromptTuningConfig19mInputTokenExtraIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor18PromptTuningConfig19mInputTokenExtraIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor18PromptTuningConfig19mInputTokenExtraIdsE"></span><span id="tensorrt_llm::executor::PromptTuningConfig::mInputTokenExtraIds__std::optional:VecTokenExtraIds:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1PromptTuningConfig_1a4c2d7366ec21819f8b579f73a1b5154b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE" title="tensorrt_llm::executor::VecTokenExtraIds"><span class="n"><span class="pre">VecTokenExtraIds</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInputTokenExtraIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig19mInputTokenExtraIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The input token extra ids for KV Cache reuse when p-tuning is enabled. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7RequestE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7RequestE"></span><span id="_CPPv2N12tensorrt_llm8executor7RequestE"></span><span id="tensorrt_llm::executor::Request"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>A class that holds information about the request. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request7RequestE9VecTokens10SizeType32bRK14SamplingConfigRK12OutputConfigRKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalI6TensorEENSt8optionalI25ExternalDraftTokensConfigEENSt8optionalI18PromptTuningConfigEENSt8optionalI15MultimodalInputEENSt8optionalI6TensorEENSt8optionalI11MropeConfigEENSt8optionalI10LoraConfigEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI22KvCacheRetentionConfigEENSt8optionalINSt6stringEEENSt8optionalI19LogitsPostProcessorEENSt8optionalI9VecTokensEENSt8optionalI6IdTypeEEb12PriorityType11RequestTypeNSt8optionalI18ContextPhaseParamsEENSt8optionalI6TensorEENSt8optionalI10SizeType32EENSt8optionalI6TensorEE10SizeType32NSt8optionalI11EagleConfigEENSt8optionalI6TensorEENSt8optionalI20GuidedDecodingParamsEENSt8optionalI10SizeType32EENSt8optionalI16MillisecondsTypeEENSt8optionalI15CacheSaltIDTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request7RequestE9VecTokens10SizeType32bRK14SamplingConfigRK12OutputConfigRKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalI6TensorEENSt8optionalI25ExternalDraftTokensConfigEENSt8optionalI18PromptTuningConfigEENSt8optionalI15MultimodalInputEENSt8optionalI6TensorEENSt8optionalI11MropeConfigEENSt8optionalI10LoraConfigEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI22KvCacheRetentionConfigEENSt8optionalINSt6stringEEENSt8optionalI19LogitsPostProcessorEENSt8optionalI9VecTokensEENSt8optionalI6IdTypeEEb12PriorityType11RequestTypeNSt8optionalI18ContextPhaseParamsEENSt8optionalI6TensorEENSt8optionalI10SizeType32EENSt8optionalI6TensorEE10SizeType32NSt8optionalI11EagleConfigEENSt8optionalI6TensorEENSt8optionalI20GuidedDecodingParamsEENSt8optionalI10SizeType32EENSt8optionalI16MillisecondsTypeEENSt8optionalI15CacheSaltIDTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request7RequestE9VecTokens10SizeType32bRK14SamplingConfigRK12OutputConfigRKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalI6TensorEENSt8optionalI25ExternalDraftTokensConfigEENSt8optionalI18PromptTuningConfigEENSt8optionalI15MultimodalInputEENSt8optionalI6TensorEENSt8optionalI11MropeConfigEENSt8optionalI10LoraConfigEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI22KvCacheRetentionConfigEENSt8optionalINSt6stringEEENSt8optionalI19LogitsPostProcessorEENSt8optionalI9VecTokensEENSt8optionalI6IdTypeEEb12PriorityType11RequestTypeNSt8optionalI18ContextPhaseParamsEENSt8optionalI6TensorEENSt8optionalI10SizeType32EENSt8optionalI6TensorEE10SizeType32NSt8optionalI11EagleConfigEENSt8optionalI6TensorEENSt8optionalI20GuidedDecodingParamsEENSt8optionalI10SizeType32EENSt8optionalI16MillisecondsTypeEENSt8optionalI15CacheSaltIDTypeEE"></span><span id="tensorrt_llm::executor::Request::Request__VecTokens.SizeType32.b.SamplingConfigCR.OutputConfigCR.std::optional:SizeType32:CR.std::optional:SizeType32:CR.std::optional:std::vector:SizeType32::.std::optional:std::list:VecTokens::.std::optional:std::list:VecTokens::.std::optional:Tensor:.std::optional:ExternalDraftTokensConfig:.std::optional:PromptTuningConfig:.std::optional:MultimodalInput:.std::optional:Tensor:.std::optional:MropeConfig:.std::optional:LoraConfig:.std::optional:LookaheadDecodingConfig:.std::optional:KvCacheRetentionConfig:.std::optional:ss:.std::optional:LogitsPostProcessor:.std::optional:VecTokens:.std::optional:IdType:.b.PriorityType.RequestType.std::optional:ContextPhaseParams:.std::optional:Tensor:.std::optional:SizeType32:.std::optional:Tensor:.SizeType32.std::optional:EagleConfig:.std::optional:Tensor:.std::optional:GuidedDecodingParams:.std::optional:SizeType32:.std::optional:MillisecondsType:.std::optional:CacheSaltIDType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ac96784076cd1ab184319571838f497bb"></span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">inputTokenIds</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokens</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">streaming</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">outputConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">endId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">padId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">positionIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">badWords</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stopWords</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">embeddingBias</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">externalDraftTokensConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pTuningConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multimodalInput</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">multimodalEmbedding</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">mRopeConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">loraConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">lookaheadConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvCacheRetentionConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">logitsPostProcessorName</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE" title="tensorrt_llm::executor::LogitsPostProcessor"><span class="n"><span class="pre">LogitsPostProcessor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">logitsPostProcessor</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">encoderInputTokenIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">clientId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnAllGeneratedTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE" title="tensorrt_llm::executor::PriorityType"><span class="n"><span class="pre">PriorityType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7Request16kDefaultPriorityE" title="tensorrt_llm::executor::Request::kDefaultPriority"><span class="n"><span class="pre">kDefaultPriority</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE" title="tensorrt_llm::executor::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE" title="tensorrt_llm::executor::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE" title="tensorrt_llm::executor::RequestType::REQUEST_TYPE_CONTEXT_AND_GENERATION"><span class="n"><span class="pre">REQUEST_TYPE_CONTEXT_AND_GENERATION</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">contextPhaseParams</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">encoderInputFeatures</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">encoderOutputLength</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">crossAttentionMask</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numReturnSequences</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">eagleConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">skipCrossAttnBlocks</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">guidedDecodingParams</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">languageAdapterUid</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE" title="tensorrt_llm::executor::MillisecondsType"><span class="n"><span class="pre">MillisecondsType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">allottedTimeMs</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE" title="tensorrt_llm::executor::CacheSaltIDType"><span class="n"><span class="pre">CacheSaltIDType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">cacheSaltID</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestE9VecTokens10SizeType32bRK14SamplingConfigRK12OutputConfigRKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalI6TensorEENSt8optionalI25ExternalDraftTokensConfigEENSt8optionalI18PromptTuningConfigEENSt8optionalI15MultimodalInputEENSt8optionalI6TensorEENSt8optionalI11MropeConfigEENSt8optionalI10LoraConfigEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI22KvCacheRetentionConfigEENSt8optionalINSt6stringEEENSt8optionalI19LogitsPostProcessorEENSt8optionalI9VecTokensEENSt8optionalI6IdTypeEEb12PriorityType11RequestTypeNSt8optionalI18ContextPhaseParamsEENSt8optionalI6TensorEENSt8optionalI10SizeType32EENSt8optionalI6TensorEE10SizeType32NSt8optionalI11EagleConfigEENSt8optionalI6TensorEENSt8optionalI20GuidedDecodingParamsEENSt8optionalI10SizeType32EENSt8optionalI16MillisecondsTypeEENSt8optionalI15CacheSaltIDTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1Request"><span class="std std-ref">Request</span></a> constructor. </p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>inputTokenIds</strong> – The input token ids </p></li>
|
||
<li><p><strong>maxTokens</strong> – The maximum number of tokens to generate </p></li>
|
||
<li><p><strong>streaming</strong> – Indicates if the responses should be streamed or not. Default is false. </p></li>
|
||
<li><p><strong>samplingConfig</strong> – The sampling configuration </p></li>
|
||
<li><p><strong>outputConfig</strong> – The output configuration </p></li>
|
||
<li><p><strong>endId</strong> – The end token id </p></li>
|
||
<li><p><strong>padId</strong> – The pad token id </p></li>
|
||
<li><p><strong>positionIds</strong> – The input position ids </p></li>
|
||
<li><p><strong>badWords</strong> – A list of bad words tokens. Each “word” can be composed of multiple tokens </p></li>
|
||
<li><p><strong>stopWords</strong> – A list of stop words tokens. Each “word” can be composed of multiple tokens </p></li>
|
||
<li><p><strong>embeddingBias</strong> – The embedding bias tensor. Expected shape is [vocab_size] </p></li>
|
||
<li><p><strong>externalDraftTokensConfig</strong> – The speculative decoding with external draft tokens configuration </p></li>
|
||
<li><p><strong>pTuningConfig</strong> – The prompt tuning configuration </p></li>
|
||
<li><p><strong>multimodalInput</strong> – The multimodal input {multimodalHashes, multimodalPositions, multimodalLengths} </p></li>
|
||
<li><p><strong>multimodalEmbedding</strong> – The multimodal embedding tensor. Expected shape is [num_multimodal_tokens, hidden_dim] </p></li>
|
||
<li><p><strong>mRopeConfig</strong> – The mrope configuration </p></li>
|
||
<li><p><strong>loraConfig</strong> – The LoRA configuration </p></li>
|
||
<li><p><strong>lookaheadConfig</strong> – The lookahead speculative decoding configuration </p></li>
|
||
<li><p><strong>kvCacheRetentionConfig</strong> – The configuration used for KV cache block eviction. </p></li>
|
||
<li><p><strong>logitsPostProcessorName</strong> – The logits postprocessor name. Must correspond to one of the logits postprocessor name provided to the <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1ExecutorConfig"><span class="std std-ref">ExecutorConfig</span></a>. </p></li>
|
||
<li><p><strong>logitsPostProcessor</strong> – The logits postprocessor dynamically specified per request; only supported with replicate=false or no tensor parallelism. </p></li>
|
||
<li><p><strong>encoderInputTokenIds</strong> – The encoder input token ids for encoder-decoder models, or encoder-only models </p></li>
|
||
<li><p><strong>clientId</strong> – </p></li>
|
||
<li><p><strong>returnAllGeneratedTokens</strong> – Indicates whether to return the full beams or just the newly generated tokens after every streaming step. </p></li>
|
||
<li><p><strong>priority</strong> – Sets the execution priority of this request. </p></li>
|
||
<li><p><strong>type</strong> – Indicate the request type for disaggregated serving mode. </p></li>
|
||
<li><p><strong>contextPhaseParams</strong> – Generated token ID from context only executor. </p></li>
|
||
<li><p><strong>encoderInputFeatures</strong> – Encoder input features for multimodal models. </p></li>
|
||
<li><p><strong>encoderOutputLength</strong> – Encoder output length if encoder input and output have different lengths (due to convolution down-sampling, etc.) </p></li>
|
||
<li><p><strong>crossAttentionMask</strong> – Cross attention mask. </p></li>
|
||
<li><p><strong>numReturnSequences</strong> – The number of returning sequences. </p></li>
|
||
<li><p><strong>eagleConfig</strong> – The EAGLE speculative decoding configuration </p></li>
|
||
<li><p><strong>skipCrossAttnBlocks</strong> – Skip the cross attention transformer blocks or not. </p></li>
|
||
<li><p><strong>guidedDecodingParams</strong> – The guided decoding parameters. </p></li>
|
||
<li><p><strong>languageAdapterUid</strong> – Task Uid for language adapter. </p></li>
|
||
<li><p><strong>allottedTimeMs</strong> – The allotted time in milliseconds after which the request is cancelled with a timedOut finish reason. The request may exceed this time slightly, but at most by 1 forward pass (in pipeline parallelism that may involve multiple micro-batches). A request can be timed-out before ever being scheduled. </p></li>
|
||
<li><p><strong>cacheSaltID</strong> – Salt ID for KV cache blocks to limit the kv cache reuse to the requests with the same string. </p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request7RequestERK7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request7RequestERK7Request"></span><span id="_CPPv2N12tensorrt_llm8executor7Request7RequestERK7Request"></span><span id="tensorrt_llm::executor::Request::Request__RequestCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ac1ba647ff89d0de162cb6b7e8a3b3218"></span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERK7Request" title="tensorrt_llm::executor::Request::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERK7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request7RequestERR7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request7RequestERR7Request"></span><span id="_CPPv2N12tensorrt_llm8executor7Request7RequestERR7Request"></span><span id="tensorrt_llm::executor::Request::Request__RequestRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aea6f0d55747d21a96b77e164de0dcacb"></span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERR7Request" title="tensorrt_llm::executor::Request::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERR7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7RequestaSERK7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7RequestaSERK7Request"></span><span id="_CPPv2N12tensorrt_llm8executor7RequestaSERK7Request"></span><span id="tensorrt_llm::executor::Request::assign-operator__RequestCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ad465314b462d06a4c1c4bd90d7ae35bb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7RequestaSERK7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7RequestaSERR7Request">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7RequestaSERR7Request"></span><span id="_CPPv2N12tensorrt_llm8executor7RequestaSERR7Request"></span><span id="tensorrt_llm::executor::Request::assign-operator__RequestRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a1702e650276cd70a43e88302ea24f815"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor7RequestE" title="tensorrt_llm::executor::Request"><span class="n"><span class="pre">Request</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7RequestaSERR7Request" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7RequestD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7RequestD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor7RequestD0Ev"></span><span id="tensorrt_llm::executor::Request::~Request"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ad615bd9f0869e60c95bb9dcb01ed8ed6"></span><span class="sig-name descname"><span class="n"><span class="pre">~Request</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7RequestD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request16getInputTokenIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request16getInputTokenIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request16getInputTokenIdsEv"></span><span id="tensorrt_llm::executor::Request::getInputTokenIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae5023d8d2d8cfa9bd3e3a16c76cc6819"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getInputTokenIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request16getInputTokenIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request12getMaxTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request12getMaxTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request12getMaxTokensEv"></span><span id="tensorrt_llm::executor::Request::getMaxTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a20edd026c107c8e66ba8e44b141cf8f1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request12getMaxTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request12getStreamingEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request12getStreamingEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request12getStreamingEv"></span><span id="tensorrt_llm::executor::Request::getStreamingC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a04fe14eab1bf2031220dfce904f6e736"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getStreaming</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request12getStreamingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request17getSamplingConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request17getSamplingConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request17getSamplingConfigEv"></span><span id="tensorrt_llm::executor::Request::getSamplingConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae89bc1ae8e0a108e2f74e43470631e4a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSamplingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request17getSamplingConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request15getOutputConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request15getOutputConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request15getOutputConfigEv"></span><span id="tensorrt_llm::executor::Request::getOutputConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a4fe15a7d4be498a08b8dcbe999a4241d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOutputConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request15getOutputConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request8getEndIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request8getEndIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request8getEndIdEv"></span><span id="tensorrt_llm::executor::Request::getEndIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a0ef03baab26435b1a0f52fb17921871d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEndId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request8getEndIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request8getPadIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request8getPadIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request8getPadIdEv"></span><span id="tensorrt_llm::executor::Request::getPadIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aeb2c8f786eefb6cb288f535ae0f8c38f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPadId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request8getPadIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request14getPositionIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request14getPositionIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request14getPositionIdsEv"></span><span id="tensorrt_llm::executor::Request::getPositionIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a438538a90ed116c31ba77134b891be19"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPositionIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request14getPositionIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request11getBadWordsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request11getBadWordsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request11getBadWordsEv"></span><span id="tensorrt_llm::executor::Request::getBadWordsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aa2203882fac60bc972d3190f5a952cbf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBadWords</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request11getBadWordsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request12getStopWordsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request12getStopWordsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request12getStopWordsEv"></span><span id="tensorrt_llm::executor::Request::getStopWordsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a0ed1ed7425f09806e0d42373ba0b1a23"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getStopWords</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request12getStopWordsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request16getEmbeddingBiasEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request16getEmbeddingBiasEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request16getEmbeddingBiasEv"></span><span id="tensorrt_llm::executor::Request::getEmbeddingBiasC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aa090ed0051151568052c4b8a662ac38e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEmbeddingBias</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request16getEmbeddingBiasEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request28getExternalDraftTokensConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request28getExternalDraftTokensConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request28getExternalDraftTokensConfigEv"></span><span id="tensorrt_llm::executor::Request::getExternalDraftTokensConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a575e5568daa7b708d2e49d3c0e27f4ca"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getExternalDraftTokensConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request28getExternalDraftTokensConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request21getPromptTuningConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request21getPromptTuningConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request21getPromptTuningConfigEv"></span><span id="tensorrt_llm::executor::Request::getPromptTuningConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1add08e33a6e9719b85ab0ddfb288c62d2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPromptTuningConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request21getPromptTuningConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request18getMultimodalInputEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request18getMultimodalInputEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request18getMultimodalInputEv"></span><span id="tensorrt_llm::executor::Request::getMultimodalInputC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a614eadec8b4a146f625bebc8e2d9108d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultimodalInput</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request18getMultimodalInputEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request22getMultimodalEmbeddingEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request22getMultimodalEmbeddingEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request22getMultimodalEmbeddingEv"></span><span id="tensorrt_llm::executor::Request::getMultimodalEmbeddingC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a77ea94602c1e7258176ae9999e36bf44"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMultimodalEmbedding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request22getMultimodalEmbeddingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request14getMropeConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request14getMropeConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request14getMropeConfigEv"></span><span id="tensorrt_llm::executor::Request::getMropeConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a2b984d67f3eb7f4a73044416f615fb8f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMropeConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request14getMropeConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request13getLoraConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request13getLoraConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request13getLoraConfigEv"></span><span id="tensorrt_llm::executor::Request::getLoraConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a358e77b87c918210c398bf78a98cafda"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLoraConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request13getLoraConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request18getLookaheadConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request18getLookaheadConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request18getLookaheadConfigEv"></span><span id="tensorrt_llm::executor::Request::getLookaheadConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a1380dab44644b7b7b267d6b3309fba01"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLookaheadConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request18getLookaheadConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request25getKvCacheRetentionConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request25getKvCacheRetentionConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request25getKvCacheRetentionConfigEv"></span><span id="tensorrt_llm::executor::Request::getKvCacheRetentionConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a8bc8c18d0bf4ee04c1b0df6e0f497ec7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKvCacheRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request25getKvCacheRetentionConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request26getLogitsPostProcessorNameEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request26getLogitsPostProcessorNameEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request26getLogitsPostProcessorNameEv"></span><span id="tensorrt_llm::executor::Request::getLogitsPostProcessorNameC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a29a2a56fef482187e36fcb41238f238f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogitsPostProcessorName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request26getLogitsPostProcessorNameEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request22getLogitsPostProcessorEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request22getLogitsPostProcessorEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request22getLogitsPostProcessorEv"></span><span id="tensorrt_llm::executor::Request::getLogitsPostProcessorC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a4240feafb83a91450c003c244d3ac919"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE" title="tensorrt_llm::executor::LogitsPostProcessor"><span class="n"><span class="pre">LogitsPostProcessor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogitsPostProcessor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request22getLogitsPostProcessorEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputTokenIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request23getEncoderInputTokenIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request23getEncoderInputTokenIdsEv"></span><span id="tensorrt_llm::executor::Request::getEncoderInputTokenIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a31dcafcde78f9af38928b038dfd7e8a8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEncoderInputTokenIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputTokenIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request11getClientIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request11getClientIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request11getClientIdEv"></span><span id="tensorrt_llm::executor::Request::getClientIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a5b78ea8903ede19e5a2991128146a40c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getClientId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request11getClientIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request11getPriorityEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request11getPriorityEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request11getPriorityEv"></span><span id="tensorrt_llm::executor::Request::getPriorityC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ace036f9c39b9ec797146b67b97c7f454"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE" title="tensorrt_llm::executor::PriorityType"><span class="n"><span class="pre">PriorityType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPriority</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request11getPriorityEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request27getReturnAllGeneratedTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request27getReturnAllGeneratedTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request27getReturnAllGeneratedTokensEv"></span><span id="tensorrt_llm::executor::Request::getReturnAllGeneratedTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a9469704f8ae0d13943fc85cc55473893"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getReturnAllGeneratedTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request27getReturnAllGeneratedTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request21getContextPhaseParamsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request21getContextPhaseParamsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request21getContextPhaseParamsEv"></span><span id="tensorrt_llm::executor::Request::getContextPhaseParamsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a6a81263006f5f937f9692e8da3e6965c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getContextPhaseParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request21getContextPhaseParamsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputFeaturesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request23getEncoderInputFeaturesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request23getEncoderInputFeaturesEv"></span><span id="tensorrt_llm::executor::Request::getEncoderInputFeaturesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ab006fa4cfd9d17df1517ccf26c958ef7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEncoderInputFeatures</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputFeaturesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request22getEncoderOutputLengthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request22getEncoderOutputLengthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request22getEncoderOutputLengthEv"></span><span id="tensorrt_llm::executor::Request::getEncoderOutputLengthC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1adb813f19b1b1d66456b412561166d0ab"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEncoderOutputLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request22getEncoderOutputLengthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request21getCrossAttentionMaskEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request21getCrossAttentionMaskEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request21getCrossAttentionMaskEv"></span><span id="tensorrt_llm::executor::Request::getCrossAttentionMaskC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ad3c31467a350829ab39b23402d4ca443"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCrossAttentionMask</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request21getCrossAttentionMaskEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request14getRequestTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request14getRequestTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request14getRequestTypeEv"></span><span id="tensorrt_llm::executor::Request::getRequestTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a1006ee8f28e9abaec817bba470f6be6a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE" title="tensorrt_llm::executor::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRequestType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request14getRequestTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request14getEagleConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request14getEagleConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request14getEagleConfigEv"></span><span id="tensorrt_llm::executor::Request::getEagleConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a8586e97f346691fb5a3cb04917fb2c49"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEagleConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request14getEagleConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request22getSkipCrossAttnBlocksEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request22getSkipCrossAttnBlocksEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request22getSkipCrossAttnBlocksEv"></span><span id="tensorrt_llm::executor::Request::getSkipCrossAttnBlocksC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a530aa5a065cd0c88f8fbfdfdf9fc58f6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSkipCrossAttnBlocks</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request22getSkipCrossAttnBlocksEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request23getGuidedDecodingParamsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request23getGuidedDecodingParamsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request23getGuidedDecodingParamsEv"></span><span id="tensorrt_llm::executor::Request::getGuidedDecodingParamsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a621f06d22af41f32ad93bce46320979f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGuidedDecodingParams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request23getGuidedDecodingParamsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request21getLanguageAdapterUidEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request21getLanguageAdapterUidEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request21getLanguageAdapterUidEv"></span><span id="tensorrt_llm::executor::Request::getLanguageAdapterUidC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aa231204e88a0a1a787e203097d6e3d96"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLanguageAdapterUid</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request21getLanguageAdapterUidEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request17getAllottedTimeMsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request17getAllottedTimeMsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request17getAllottedTimeMsEv"></span><span id="tensorrt_llm::executor::Request::getAllottedTimeMsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a9cb91f2565d9973a5533b6f377673193"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE" title="tensorrt_llm::executor::MillisecondsType"><span class="n"><span class="pre">MillisecondsType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAllottedTimeMs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request17getAllottedTimeMsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request14getCacheSaltIDEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request14getCacheSaltIDEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request14getCacheSaltIDEv"></span><span id="tensorrt_llm::executor::Request::getCacheSaltIDC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a289f580bbf1029434c8f1cc888e1ea02"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE" title="tensorrt_llm::executor::CacheSaltIDType"><span class="n"><span class="pre">CacheSaltIDType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCacheSaltID</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request14getCacheSaltIDEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor7Request24getAdditionalOutputNamesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor7Request24getAdditionalOutputNamesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor7Request24getAdditionalOutputNamesEv"></span><span id="tensorrt_llm::executor::Request::getAdditionalOutputNamesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a9db754f3cc591b6c7948563a76d24873"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAdditionalOutputNames</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor7Request24getAdditionalOutputNamesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request12setStreamingEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request12setStreamingEb"></span><span id="_CPPv2N12tensorrt_llm8executor7Request12setStreamingEb"></span><span id="tensorrt_llm::executor::Request::setStreaming__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae68a921ab34759ffc097e342253d52f9"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setStreaming</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">streaming</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request12setStreamingEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request17setSamplingConfigERK14SamplingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request17setSamplingConfigERK14SamplingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request17setSamplingConfigERK14SamplingConfig"></span><span id="tensorrt_llm::executor::Request::setSamplingConfig__SamplingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1abe649c4845388cea78bf22c291538341"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSamplingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request17setSamplingConfigERK14SamplingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request15setOutputConfigERK12OutputConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request15setOutputConfigERK12OutputConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request15setOutputConfigERK12OutputConfig"></span><span id="tensorrt_llm::executor::Request::setOutputConfig__OutputConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a73a7ec9973c9a76faf89738aedaa69f2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setOutputConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE" title="tensorrt_llm::executor::OutputConfig"><span class="n"><span class="pre">OutputConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">outputConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request15setOutputConfigERK12OutputConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request8setEndIdE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request8setEndIdE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor7Request8setEndIdE10SizeType32"></span><span id="tensorrt_llm::executor::Request::setEndId__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a76a2cc1cc476cb0a050c24a73a6e5cbc"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEndId</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">endId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request8setEndIdE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request8setPadIdE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request8setPadIdE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor7Request8setPadIdE10SizeType32"></span><span id="tensorrt_llm::executor::Request::setPadId__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a8f1d378a89c237f2c79f5510d285a32e"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPadId</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">padId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request8setPadIdE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request14setPositionIdsERKNSt6vectorI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request14setPositionIdsERKNSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request14setPositionIdsERKNSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::executor::Request::setPositionIds__std::vector:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1afb5b1b10b3d2976d9f40320c63614122"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPositionIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">positionIds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request14setPositionIdsERKNSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request11setBadWordsERKNSt4listI9VecTokensEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request11setBadWordsERKNSt4listI9VecTokensEE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request11setBadWordsERKNSt4listI9VecTokensEE"></span><span id="tensorrt_llm::executor::Request::setBadWords__std::list:VecTokens:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae2c911bc838c38ff9c0955063d532b5b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBadWords</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">badWords</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request11setBadWordsERKNSt4listI9VecTokensEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request12setStopWordsERKNSt4listI9VecTokensEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request12setStopWordsERKNSt4listI9VecTokensEE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request12setStopWordsERKNSt4listI9VecTokensEE"></span><span id="tensorrt_llm::executor::Request::setStopWords__std::list:VecTokens:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ac3fab641b66477d4c11446a7a09f9f72"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setStopWords</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">stopWords</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request12setStopWordsERKNSt4listI9VecTokensEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request16setEmbeddingBiasERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request16setEmbeddingBiasERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor7Request16setEmbeddingBiasERK6Tensor"></span><span id="tensorrt_llm::executor::Request::setEmbeddingBias__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a3184ce8fb36c13840dec0e219056e3e1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEmbeddingBias</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">embeddingBias</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request16setEmbeddingBiasERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request28setExternalDraftTokensConfigERK25ExternalDraftTokensConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request28setExternalDraftTokensConfigERK25ExternalDraftTokensConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request28setExternalDraftTokensConfigERK25ExternalDraftTokensConfig"></span><span id="tensorrt_llm::executor::Request::setExternalDraftTokensConfig__ExternalDraftTokensConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a311a94236786522ab77f47fe38bd0402"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setExternalDraftTokensConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">externalDraftTokensConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request28setExternalDraftTokensConfigERK25ExternalDraftTokensConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request21setPromptTuningConfigERK18PromptTuningConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request21setPromptTuningConfigERK18PromptTuningConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request21setPromptTuningConfigERK18PromptTuningConfig"></span><span id="tensorrt_llm::executor::Request::setPromptTuningConfig__PromptTuningConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ad01f7cf40b183ad8b7c77ee0c03d8c51"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPromptTuningConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE" title="tensorrt_llm::executor::PromptTuningConfig"><span class="n"><span class="pre">PromptTuningConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">pTuningConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request21setPromptTuningConfigERK18PromptTuningConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request22setMultimodalEmbeddingERK6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request22setMultimodalEmbeddingERK6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor7Request22setMultimodalEmbeddingERK6Tensor"></span><span id="tensorrt_llm::executor::Request::setMultimodalEmbedding__TensorCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aed1946173f299ff493f98f0fd0fd8a42"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMultimodalEmbedding</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">multimodalEmbedding</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request22setMultimodalEmbeddingERK6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request18setMultimodalInputERK15MultimodalInput">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request18setMultimodalInputERK15MultimodalInput"></span><span id="_CPPv2N12tensorrt_llm8executor7Request18setMultimodalInputERK15MultimodalInput"></span><span id="tensorrt_llm::executor::Request::setMultimodalInput__MultimodalInputCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ab9763068bbff784beb3b5a9e70b02626"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMultimodalInput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE" title="tensorrt_llm::executor::MultimodalInput"><span class="n"><span class="pre">MultimodalInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">multimodalInput</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request18setMultimodalInputERK15MultimodalInput" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request14setMropeConfigERK11MropeConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request14setMropeConfigERK11MropeConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request14setMropeConfigERK11MropeConfig"></span><span id="tensorrt_llm::executor::Request::setMropeConfig__MropeConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae2a2b91a7a51f369b93965cb8ca4e479"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMropeConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE" title="tensorrt_llm::executor::MropeConfig"><span class="n"><span class="pre">MropeConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">mRopeConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request14setMropeConfigERK11MropeConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request13setLoraConfigERK10LoraConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request13setLoraConfigERK10LoraConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request13setLoraConfigERK10LoraConfig"></span><span id="tensorrt_llm::executor::Request::setLoraConfig__LoraConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ada0604e9f9989519d8595e5b0777d875"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLoraConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE" title="tensorrt_llm::executor::LoraConfig"><span class="n"><span class="pre">LoraConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">loraConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request13setLoraConfigERK10LoraConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request18setLookaheadConfigERK23LookaheadDecodingConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request18setLookaheadConfigERK23LookaheadDecodingConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request18setLookaheadConfigERK23LookaheadDecodingConfig"></span><span id="tensorrt_llm::executor::Request::setLookaheadConfig__LookaheadDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1aebd7ed087f0e1ebeb03edd9b44d3a28d"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLookaheadConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lookaheadConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request18setLookaheadConfigERK23LookaheadDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request25setKvCacheRetentionConfigERK22KvCacheRetentionConfig">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request25setKvCacheRetentionConfigERK22KvCacheRetentionConfig"></span><span id="_CPPv2N12tensorrt_llm8executor7Request25setKvCacheRetentionConfigERK22KvCacheRetentionConfig"></span><span id="tensorrt_llm::executor::Request::setKvCacheRetentionConfig__KvCacheRetentionConfigCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1acc2a684dd429a6e6a2f738b597ce3dad"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setKvCacheRetentionConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE" title="tensorrt_llm::executor::KvCacheRetentionConfig"><span class="n"><span class="pre">KvCacheRetentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">kvCacheRetentionConfig</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request25setKvCacheRetentionConfigERK22KvCacheRetentionConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request26setLogitsPostProcessorNameERKNSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request26setLogitsPostProcessorNameERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request26setLogitsPostProcessorNameERKNSt6stringE"></span><span id="tensorrt_llm::executor::Request::setLogitsPostProcessorName__ssCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a26704f9863977008bd456b1aa49e3e55"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLogitsPostProcessorName</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">logitsPostProcessorName</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request26setLogitsPostProcessorNameERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request22setLogitsPostProcessorERKNSt8optionalI19LogitsPostProcessorEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request22setLogitsPostProcessorERKNSt8optionalI19LogitsPostProcessorEE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request22setLogitsPostProcessorERKNSt8optionalI19LogitsPostProcessorEE"></span><span id="tensorrt_llm::executor::Request::setLogitsPostProcessor__std::optional:LogitsPostProcessor:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ac608a4745474f5376ff8d7f87e36117c"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLogitsPostProcessor</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE" title="tensorrt_llm::executor::LogitsPostProcessor"><span class="n"><span class="pre">LogitsPostProcessor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">logitsPostProcessor</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request22setLogitsPostProcessorERKNSt8optionalI19LogitsPostProcessorEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputTokenIdsERK9VecTokens">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request23setEncoderInputTokenIdsERK9VecTokens"></span><span id="_CPPv2N12tensorrt_llm8executor7Request23setEncoderInputTokenIdsERK9VecTokens"></span><span id="tensorrt_llm::executor::Request::setEncoderInputTokenIds__VecTokensCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1adf41e2b961d2bd01a43ea7ef17d19a01"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEncoderInputTokenIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE" title="tensorrt_llm::executor::VecTokens"><span class="n"><span class="pre">VecTokens</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">encoderInputTokenIds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputTokenIdsERK9VecTokens" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request11setClientIdE6IdType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request11setClientIdE6IdType"></span><span id="_CPPv2N12tensorrt_llm8executor7Request11setClientIdE6IdType"></span><span id="tensorrt_llm::executor::Request::setClientId__IdType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1abf1a51a1a1462745563462e323e725bd"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setClientId</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">clientId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request11setClientIdE6IdType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request11setPriorityE12PriorityType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request11setPriorityE12PriorityType"></span><span id="_CPPv2N12tensorrt_llm8executor7Request11setPriorityE12PriorityType"></span><span id="tensorrt_llm::executor::Request::setPriority__PriorityType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1acfcea4db0cf194766c5fde1cf4ec5e77"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPriority</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE" title="tensorrt_llm::executor::PriorityType"><span class="n"><span class="pre">PriorityType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request11setPriorityE12PriorityType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request27setReturnAllGeneratedTokensEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request27setReturnAllGeneratedTokensEb"></span><span id="_CPPv2N12tensorrt_llm8executor7Request27setReturnAllGeneratedTokensEb"></span><span id="tensorrt_llm::executor::Request::setReturnAllGeneratedTokens__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a122fe4b9e64de4c5305663d1fc0711a6"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setReturnAllGeneratedTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">returnAllGeneratedTokens</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request27setReturnAllGeneratedTokensEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request14setRequestTypeERK11RequestType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request14setRequestTypeERK11RequestType"></span><span id="_CPPv2N12tensorrt_llm8executor7Request14setRequestTypeERK11RequestType"></span><span id="tensorrt_llm::executor::Request::setRequestType__RequestTypeCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1af880e937db1156c88361d1fe3b9e1ce1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRequestType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE" title="tensorrt_llm::executor::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">requestType</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request14setRequestTypeERK11RequestType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request21setContextPhaseParamsE18ContextPhaseParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request21setContextPhaseParamsE18ContextPhaseParams"></span><span id="_CPPv2N12tensorrt_llm8executor7Request21setContextPhaseParamsE18ContextPhaseParams"></span><span id="tensorrt_llm::executor::Request::setContextPhaseParams__ContextPhaseParams"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1adf0ae5e8364ea484c784c018a0521aab"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setContextPhaseParams</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">contextPhaseParams</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request21setContextPhaseParamsE18ContextPhaseParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputFeaturesE6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request23setEncoderInputFeaturesE6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor7Request23setEncoderInputFeaturesE6Tensor"></span><span id="tensorrt_llm::executor::Request::setEncoderInputFeatures__Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ac9b8af7d68183782ae71c49fa1d417ec"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEncoderInputFeatures</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">encoderInputFeatures</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputFeaturesE6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request22setEncoderOutputLengthE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request22setEncoderOutputLengthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor7Request22setEncoderOutputLengthE10SizeType32"></span><span id="tensorrt_llm::executor::Request::setEncoderOutputLength__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a4b485629537541897b7a5cac3b5a8da0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEncoderOutputLength</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">encoderOutputLength</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request22setEncoderOutputLengthE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request21setCrossAttentionMaskE6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request21setCrossAttentionMaskE6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor7Request21setCrossAttentionMaskE6Tensor"></span><span id="tensorrt_llm::executor::Request::setCrossAttentionMask__Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1addc7a8f4af939975c638d46c25d8c04b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCrossAttentionMask</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">crossAttentionMask</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request21setCrossAttentionMaskE6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request14setEagleConfigERKNSt8optionalI11EagleConfigEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request14setEagleConfigERKNSt8optionalI11EagleConfigEE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request14setEagleConfigERKNSt8optionalI11EagleConfigEE"></span><span id="tensorrt_llm::executor::Request::setEagleConfig__std::optional:EagleConfig:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1acb170f755a284ae5d474d6a23ef20faf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEagleConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">eagleConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request14setEagleConfigERKNSt8optionalI11EagleConfigEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request22setSkipCrossAttnBlocksE6Tensor">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request22setSkipCrossAttnBlocksE6Tensor"></span><span id="_CPPv2N12tensorrt_llm8executor7Request22setSkipCrossAttnBlocksE6Tensor"></span><span id="tensorrt_llm::executor::Request::setSkipCrossAttnBlocks__Tensor"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1af0d79c0e179a1e5274a75cc61a2f1fb2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSkipCrossAttnBlocks</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">skipCrossAttnBlocks</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request22setSkipCrossAttnBlocksE6Tensor" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request23setGuidedDecodingParamsERK20GuidedDecodingParams">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request23setGuidedDecodingParamsERK20GuidedDecodingParams"></span><span id="_CPPv2N12tensorrt_llm8executor7Request23setGuidedDecodingParamsERK20GuidedDecodingParams"></span><span id="tensorrt_llm::executor::Request::setGuidedDecodingParams__GuidedDecodingParamsCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a0c0331051ef7d1a0f3a469c1671993b7"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setGuidedDecodingParams</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE" title="tensorrt_llm::executor::GuidedDecodingParams"><span class="n"><span class="pre">GuidedDecodingParams</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">guidedDecodingParams</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request23setGuidedDecodingParamsERK20GuidedDecodingParams" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request21setLanguageAdapterUidE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request21setLanguageAdapterUidE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor7Request21setLanguageAdapterUidE10SizeType32"></span><span id="tensorrt_llm::executor::Request::setLanguageAdapterUid__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a90bf728cbde2bf9c1c383559f1e660ee"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLanguageAdapterUid</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">languageAdapterUid</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request21setLanguageAdapterUidE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request17setAllottedTimeMsE16MillisecondsType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request17setAllottedTimeMsE16MillisecondsType"></span><span id="_CPPv2N12tensorrt_llm8executor7Request17setAllottedTimeMsE16MillisecondsType"></span><span id="tensorrt_llm::executor::Request::setAllottedTimeMs__MillisecondsType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a384ac064c96a053a479d4213c35a1b48"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setAllottedTimeMs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE" title="tensorrt_llm::executor::MillisecondsType"><span class="n"><span class="pre">MillisecondsType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">allottedTimeMs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request17setAllottedTimeMsE16MillisecondsType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request14setCacheSaltIDE15CacheSaltIDType">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request14setCacheSaltIDE15CacheSaltIDType"></span><span id="_CPPv2N12tensorrt_llm8executor7Request14setCacheSaltIDE15CacheSaltIDType"></span><span id="tensorrt_llm::executor::Request::setCacheSaltID__CacheSaltIDType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a0238105137443e9213206de1985b17c1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCacheSaltID</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE" title="tensorrt_llm::executor::CacheSaltIDType"><span class="n"><span class="pre">CacheSaltIDType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cacheSaltID</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request14setCacheSaltIDE15CacheSaltIDType" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request16kDefaultPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request16kDefaultPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request16kDefaultPriorityE"></span><span id="tensorrt_llm::executor::Request::kDefaultPriority__PriorityType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1af72bb695ce2e2024cb6dbf32e788a219"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE" title="tensorrt_llm::executor::PriorityType"><span class="n"><span class="pre">PriorityType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultPriority</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0.5</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request16kDefaultPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request25kBatchedPostProcessorNameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request25kBatchedPostProcessorNameE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request25kBatchedPostProcessorNameE"></span><span id="tensorrt_llm::executor::Request::kBatchedPostProcessorName__auto"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1add106668c254a78b6bc7432fcc08913e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBatchedPostProcessorName</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">"batched"</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request25kBatchedPostProcessorNameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>This logits postprocessor name will dispatch to the batched logits postprocessor. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request31kDynamicPostProcessorNamePrefixE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request31kDynamicPostProcessorNamePrefixE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request31kDynamicPostProcessorNamePrefixE"></span><span id="tensorrt_llm::executor::Request::kDynamicPostProcessorNamePrefix__auto"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1ae805282e2755305cf030fc9605361d46"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDynamicPostProcessorNamePrefix</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">"dynamic"</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request31kDynamicPostProcessorNamePrefixE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Dynamic logits postprocessor name will be “dynamic” + requestId. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor7Request5mImplE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor7Request5mImplE"></span><span id="_CPPv2N12tensorrt_llm8executor7Request5mImplE"></span><span id="tensorrt_llm::executor::Request::mImpl__std::unique_ptr:Impl:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Request_1a2bef0e71b0eadc746b10462a419ff339"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Impl</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mImpl</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor7Request5mImplE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ResponseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ResponseE"></span><span id="_CPPv2N12tensorrt_llm8executor8ResponseE"></span><span id="tensorrt_llm::executor::Response"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Response</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Class that holds either an error or a result. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdTypeNSt6stringENSt8optionalI6IdTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Response8ResponseE6IdTypeNSt6stringENSt8optionalI6IdTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Response8ResponseE6IdTypeNSt6stringENSt8optionalI6IdTypeEE"></span><span id="tensorrt_llm::executor::Response::Response__IdType.ss.std::optional:IdType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1ac1c11ef972edc9ed1d4ee378b43592f7"></span><span class="sig-name descname"><span class="n"><span class="pre">Response</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">requestId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">errorMsg</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">clientId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdTypeNSt6stringENSt8optionalI6IdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE"></span><span id="tensorrt_llm::executor::Response::Response__IdType.Result.std::optional:IdType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a9c1ffcd57a905446841ab9379e514abe"></span><span class="sig-name descname"><span class="n"><span class="pre">Response</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">requestId</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE" title="tensorrt_llm::executor::Response::Response::Result"><span class="n"><span class="pre">Result</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">Result</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">clientId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ResponseD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ResponseD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8ResponseD0Ev"></span><span id="tensorrt_llm::executor::Response::~Response"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1aef44ea215e23b7f97c69b3de2e2d612b"></span><span class="sig-name descname"><span class="n"><span class="pre">~Response</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ResponseD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Response8ResponseERK8Response">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Response8ResponseERK8Response"></span><span id="_CPPv2N12tensorrt_llm8executor8Response8ResponseERK8Response"></span><span id="tensorrt_llm::executor::Response::Response__ResponseCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1aa59cf06a2cd20bcbb414c6329f950653"></span><span class="sig-name descname"><span class="n"><span class="pre">Response</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERK8Response" title="tensorrt_llm::executor::Response::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERK8Response" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Response8ResponseERR8Response">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Response8ResponseERR8Response"></span><span id="_CPPv2N12tensorrt_llm8executor8Response8ResponseERR8Response"></span><span id="tensorrt_llm::executor::Response::Response__ResponseRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a6775c21a72390415490ec5138a4c91c3"></span><span class="sig-name descname"><span class="n"><span class="pre">Response</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERR8Response" title="tensorrt_llm::executor::Response::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERR8Response" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ResponseaSERK8Response">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ResponseaSERK8Response"></span><span id="_CPPv2N12tensorrt_llm8executor8ResponseaSERK8Response"></span><span id="tensorrt_llm::executor::Response::assign-operator__ResponseCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1acda14277748b6b6f7538ff1f0636215d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ResponseaSERK8Response" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8ResponseaSERR8Response">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8ResponseaSERR8Response"></span><span id="_CPPv2N12tensorrt_llm8executor8ResponseaSERR8Response"></span><span id="tensorrt_llm::executor::Response::assign-operator__ResponseRR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a1292cbf4171fe10a8b2824daa6bb6ee4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8ResponseE" title="tensorrt_llm::executor::Response"><span class="n"><span class="pre">Response</span></span></a><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8ResponseaSERR8Response" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Response12getRequestIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Response12getRequestIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Response12getRequestIdEv"></span><span id="tensorrt_llm::executor::Response::getRequestIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1adf694176a77659e4f4afcb4fd7112eea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRequestId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Response12getRequestIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the id of the request for which this response was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Response11getClientIdEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Response11getClientIdEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Response11getClientIdEv"></span><span id="tensorrt_llm::executor::Response::getClientIdC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a0ce0d259f14b7fc6a63db97e322bfc93"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE" title="tensorrt_llm::executor::IdType"><span class="n"><span class="pre">IdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getClientId</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Response11getClientIdEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the client id of the request for which this response was generated. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Response8hasErrorEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Response8hasErrorEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Response8hasErrorEv"></span><span id="tensorrt_llm::executor::Response::hasErrorC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1acd0e0fe2d71ecd4e9a6ae7529d5a9958"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">hasError</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Response8hasErrorEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if this response has an error or not. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Response11getErrorMsgEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Response11getErrorMsgEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Response11getErrorMsgEv"></span><span id="tensorrt_llm::executor::Response::getErrorMsgC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a4eeac11a9ae40fbaeb28ef96451f5185"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getErrorMsg</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Response11getErrorMsgEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the error msg for this response Will throw an exception if hasError is false. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8Response9getResultEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8Response9getResultEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8Response9getResultEv"></span><span id="tensorrt_llm::executor::Response::getResultC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a917149b976494cf2812c77cc87f16372"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6ResultE" title="tensorrt_llm::executor::Result"><span class="n"><span class="pre">Result</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getResult</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8Response9getResultEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Get the result for this response Will throw an exception if hasResult is true. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8Response5mImplE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8Response5mImplE"></span><span id="_CPPv2N12tensorrt_llm8executor8Response5mImplE"></span><span id="tensorrt_llm::executor::Response::mImpl__std::unique_ptr:Impl:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1Response_1a62ec31b2113d81890e913e6b5e4fddac"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">Impl</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mImpl</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8Response5mImplE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6ResultE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6ResultE"></span><span id="_CPPv2N12tensorrt_llm8executor6ResultE"></span><span id="tensorrt_llm::executor::Result"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Result</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6ResultE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Struct that holds the generation result. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result7isFinalE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result7isFinalE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result7isFinalE"></span><span id="tensorrt_llm::executor::Result::isFinal__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1aa17894b95a897fc98e10059a95060097"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isFinal</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result7isFinalE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if this is the final result for the request. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result14outputTokenIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result14outputTokenIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result14outputTokenIdsE"></span><span id="tensorrt_llm::executor::Result::outputTokenIds__BeamTokens"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a878f113efb1c2e087bece4205f61fbfb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10BeamTokensE" title="tensorrt_llm::executor::BeamTokens"><span class="n"><span class="pre">BeamTokens</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outputTokenIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result14outputTokenIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The output tokens for each beam. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result11cumLogProbsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result11cumLogProbsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result11cumLogProbsE"></span><span id="tensorrt_llm::executor::Result::cumLogProbs__std::optional:VecLogProbs:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1aa4463aefb92ec56587efff7b64ca3f97"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11VecLogProbsE" title="tensorrt_llm::executor::VecLogProbs"><span class="n"><span class="pre">VecLogProbs</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result11cumLogProbsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The cumulative log probabilities. Size beamSize. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result8logProbsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result8logProbsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result8logProbsE"></span><span id="tensorrt_llm::executor::Result::logProbs__std::optional:std::vector:VecLogProbs::"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a11e7bc33f082ca62a83dbc12a734889c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11VecLogProbsE" title="tensorrt_llm::executor::VecLogProbs"><span class="n"><span class="pre">VecLogProbs</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result8logProbsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The log probabilities for each generated token. Size [beamSize, outputLen]. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result13contextLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result13contextLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result13contextLogitsE"></span><span id="tensorrt_llm::executor::Result::contextLogits__std::optional:Tensor:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1aade7284f001e6d0418709492b3b90cf1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">contextLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result13contextLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The context logits. Size [promptLen, vocabSizePadded]. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result16generationLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result16generationLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result16generationLogitsE"></span><span id="tensorrt_llm::executor::Result::generationLogits__std::optional:Tensor:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a3c7678084cfb911c75b34ea2c4043d9b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result16generationLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The generation logits. Size [beamSize, maxTokens, vocabSizePadded] (non-streaming) or [maxTokens, beamSize, vocabSizePadded] (streaming and allGeneratedTokens) or [1, beamSize, vocabSizePadded] (streaming and non-allGeneratedTokens) </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result21specDecFastLogitsInfoE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result21specDecFastLogitsInfoE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result21specDecFastLogitsInfoE"></span><span id="tensorrt_llm::executor::Result::specDecFastLogitsInfo__std::optional:SpeculativeDecodingFastLogitsInfo:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a571a13ca4077accc2f47313125b34820"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE" title="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo"><span class="n"><span class="pre">SpeculativeDecodingFastLogitsInfo</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecFastLogitsInfo</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result21specDecFastLogitsInfoE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Logits information for direct transfer when using fast logits. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result13encoderOutputE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result13encoderOutputE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result13encoderOutputE"></span><span id="tensorrt_llm::executor::Result::encoderOutput__std::optional:Tensor:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1af1fe609c6a99ac08b1bf4c687ca8e654"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">encoderOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result13encoderOutputE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The encoder output. Size [encoderLen, hiddenSize]. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result13finishReasonsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result13finishReasonsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result13finishReasonsE"></span><span id="tensorrt_llm::executor::Result::finishReasons__std::vector:FinishReason:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a71979ade4af529c950547e687f2037a1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor12FinishReasonE" title="tensorrt_llm::executor::FinishReason"><span class="n"><span class="pre">FinishReason</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finishReasons</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result13finishReasonsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The reason why the model stopped generating tokens for each beam in this request. Size [beamSize]. Currently only supported when beamSize is 1 and when using BatchingType::kINFLIGHT. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result18contextPhaseParamsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result18contextPhaseParamsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result18contextPhaseParamsE"></span><span id="tensorrt_llm::executor::Result::contextPhaseParams__std::optional:ContextPhaseParams:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a8d585cd5ac9ffc418b4f7ed8b03b6590"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE" title="tensorrt_llm::executor::ContextPhaseParams"><span class="n"><span class="pre">ContextPhaseParams</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">contextPhaseParams</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result18contextPhaseParamsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The params of the context phase. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result12decodingIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result12decodingIterE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result12decodingIterE"></span><span id="tensorrt_llm::executor::Result::decodingIter__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1ad0023b5973c19222f825d92099097469"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">decodingIter</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result12decodingIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The number of the decoding iterations used to generate the result. In autoregressive decoding, it is equal to the maximum length of the beam in outputTokenIds. In speculative decoding, might be less than maximum length of the beam in outputTokenIds as more than one token can be generated per iteration. Used for speculative decoding statistics. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result23avgDecodedTokensPerIterE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result23avgDecodedTokensPerIterE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result23avgDecodedTokensPerIterE"></span><span id="tensorrt_llm::executor::Result::avgDecodedTokensPerIter__float"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a43c439aecd73eec9d9f534fea7a9ed15"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">avgDecodedTokensPerIter</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0.0f</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result23avgDecodedTokensPerIterE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The average number of decoded tokens per iteration. For standard model it is 1. For speculative decoding model >= 1 &#8212; number of draft tokens accepted per step + 1. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result13sequenceIndexE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result13sequenceIndexE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result13sequenceIndexE"></span><span id="tensorrt_llm::executor::Result::sequenceIndex__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a8e4211d45667d1632b40c9340f60e848"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">sequenceIndex</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result13sequenceIndexE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The index of the output sequence of this result where 0 <= sequenceIndex < numReturnSequences. In beam search (beamWidth > 1), this index will be always zero because all beams to be returned are included in this result. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result15isSequenceFinalE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result15isSequenceFinalE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result15isSequenceFinalE"></span><span id="tensorrt_llm::executor::Result::isSequenceFinal__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a477ccd08f3c10499af4c223ac0e71424"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isSequenceFinal</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result15isSequenceFinalE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Indicates if this is the final result for a given sequence in the request In beam search (beamWidth > 1), the value will always equal to the value of isFinal. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result18requestPerfMetricsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result18requestPerfMetricsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result18requestPerfMetricsE"></span><span id="tensorrt_llm::executor::Result::requestPerfMetrics__std::optional:RequestPerfMetrics:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1a5108e30550019f4330573ae1b2a059fc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE" title="tensorrt_llm::executor::RequestPerfMetrics"><span class="n"><span class="pre">RequestPerfMetrics</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">requestPerfMetrics</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result18requestPerfMetricsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Performance metrics if returnPerfMetrics is set in <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1OutputConfig"><span class="std std-ref">OutputConfig</span></a>. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor6Result17additionalOutputsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor6Result17additionalOutputsE"></span><span id="_CPPv2N12tensorrt_llm8executor6Result17additionalOutputsE"></span><span id="tensorrt_llm::executor::Result::additionalOutputs__std::vector:AdditionalOutput:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1Result_1ac994b7610532dc9665d6462cc7c0f33b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE" title="tensorrt_llm::executor::AdditionalOutput"><span class="n"><span class="pre">AdditionalOutput</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">additionalOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor6Result17additionalOutputsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The additional outputs. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDurationE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor28RetentionPriorityAndDurationE"></span><span id="_CPPv2N12tensorrt_llm8executor28RetentionPriorityAndDurationE"></span><span id="tensorrt_llm::executor::RetentionPriorityAndDuration"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RetentionPriorityAndDuration"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RetentionPriorityAndDuration</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDurationE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration28RetentionPriorityAndDurationERKNSt8optionalI17RetentionPriorityEERKNSt8optionalINSt6chrono12millisecondsEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor28RetentionPriorityAndDuration28RetentionPriorityAndDurationERKNSt8optionalI17RetentionPriorityEERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="_CPPv2N12tensorrt_llm8executor28RetentionPriorityAndDuration28RetentionPriorityAndDurationERKNSt8optionalI17RetentionPriorityEERKNSt8optionalINSt6chrono12millisecondsEEE"></span><span id="tensorrt_llm::executor::RetentionPriorityAndDuration::RetentionPriorityAndDuration__std::optional:RetentionPriority:CR.std::optional:std::chrono::milliseconds:CR"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RetentionPriorityAndDuration_1a97cd54dd5c5b97062851ab77062a1abb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RetentionPriorityAndDuration</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">retentionPriority</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">durationMs</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration28RetentionPriorityAndDurationERKNSt8optionalI17RetentionPriorityEERKNSt8optionalINSt6chrono12millisecondsEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration17retentionPriorityE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor28RetentionPriorityAndDuration17retentionPriorityE"></span><span id="_CPPv2N12tensorrt_llm8executor28RetentionPriorityAndDuration17retentionPriorityE"></span><span id="tensorrt_llm::executor::RetentionPriorityAndDuration::retentionPriority__std::optional:RetentionPriority:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RetentionPriorityAndDuration_1a00d6d80567d8afbd37c114f12fda68b6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE" title="tensorrt_llm::executor::RetentionPriority"><span class="n"><span class="pre">RetentionPriority</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">retentionPriority</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration17retentionPriorityE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration10durationMsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor28RetentionPriorityAndDuration10durationMsE"></span><span id="_CPPv2N12tensorrt_llm8executor28RetentionPriorityAndDuration10durationMsE"></span><span id="tensorrt_llm::executor::RetentionPriorityAndDuration::durationMs__std::optional:std::chrono::milliseconds:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1RetentionPriorityAndDuration_1ad1906adfd260cd18f9e6855309cf4f0f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">chrono</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">milliseconds</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">durationMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration10durationMsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfigE"></span><span id="tensorrt_llm::executor::SamplingConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Sampling configuration. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14SamplingConfigE10SizeType32RKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI11TokenIdTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI14RandomSeedTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalINSt6vectorI10SizeType32EEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14SamplingConfigE10SizeType32RKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI11TokenIdTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI14RandomSeedTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalINSt6vectorI10SizeType32EEEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14SamplingConfigE10SizeType32RKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI11TokenIdTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI14RandomSeedTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalINSt6vectorI10SizeType32EEEE"></span><span id="tensorrt_llm::executor::SamplingConfig::SamplingConfig__SizeType32.std::optional:SizeType32:CR.std::optional:FloatType:CR.std::optional:FloatType:CR.std::optional:TokenIdType:CR.std::optional:FloatType:CR.std::optional:RandomSeedType:CR.std::optional:FloatType:CR.std::optional:SizeType32:CR.std::optional:FloatType:CR.std::optional:FloatType:CR.std::optional:FloatType:CR.std::optional:FloatType:CR.std::optional:SizeType32:CR.std::optional:FloatType:CR.std::optional:SizeType32:CR.std::optional:SizeType32:CR.std::optional:SizeType32:CR.std::optional:FloatType:CR.std::optional:std::vector:SizeType32::CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a818b82e58aa3791b6a55d87a4f2a8dcd"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topK</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPMin</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPResetIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPDecay</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE" title="tensorrt_llm::executor::RandomSeedType"><span class="n"><span class="pre">RandomSeedType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">seed</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">temperature</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamSearchDiversityRate</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">repetitionPenalty</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">presencePenalty</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">frequencyPenalty</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">promptIgnoreLength</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lengthPenalty</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">earlyStopping</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">noRepeatNgramSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">numReturnSequences</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamWidthArray</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14SamplingConfigE10SizeType32RKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI11TokenIdTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI14RandomSeedTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalINSt6vectorI10SizeType32EEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Constructor for <a class="reference internal" href="#classtensorrt__llm_1_1executor_1_1SamplingConfig"><span class="std std-ref">SamplingConfig</span></a> See description of parameters below. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfigeqERK14SamplingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfigeqERK14SamplingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfigeqERK14SamplingConfig"></span><span id="tensorrt_llm::executor::SamplingConfig::eq-operator__SamplingConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a08919f8eb49492f6e0f23f20b2ff0555"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfigeqERK14SamplingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getBeamWidthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig12getBeamWidthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig12getBeamWidthEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getBeamWidthC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a9f3845e5dabe1f14aa2130c7adc8dc9d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBeamWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getBeamWidthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getNumReturnBeamsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig17getNumReturnBeamsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig17getNumReturnBeamsEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getNumReturnBeamsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab12e1b657d4a05e2ab49c51d19671f28"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumReturnBeams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getNumReturnBeamsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopKEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig7getTopKEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig7getTopKEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTopKC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a2946f6278862e85ef92d085376f12cbf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTopK</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopKEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig7getTopPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig7getTopPEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTopPC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a71d2f03b643676951fa7729d0804edd1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTopP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig10getTopPMinEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig10getTopPMinEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig10getTopPMinEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTopPMinC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1afd69a9dcf61940b09f83819c8a3bf849"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTopPMin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig10getTopPMinEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig15getTopPResetIdsEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig15getTopPResetIdsEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig15getTopPResetIdsEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTopPResetIdsC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a5b077cc8420200b24a127e9bb0f84cf1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTopPResetIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig15getTopPResetIdsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getTopPDecayEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig12getTopPDecayEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig12getTopPDecayEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTopPDecayC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ae495fcf888bb96f6b252e60ea42be9a4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTopPDecay</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getTopPDecayEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getSeedEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig7getSeedEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig7getSeedEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getSeedC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a404891b17704f007b37acfcab3bd6738"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE" title="tensorrt_llm::executor::RandomSeedType"><span class="n"><span class="pre">RandomSeedType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSeed</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getSeedEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig14getTemperatureEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig14getTemperatureEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig14getTemperatureEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getTemperatureC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a8ac079c1742f930190ed0e488a0e9368"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTemperature</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig14getTemperatureEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getMinTokensEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig12getMinTokensEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig12getMinTokensEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getMinTokensC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a0c0860e9f2280c3a35fdcd6e523af493"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMinTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getMinTokensEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig26getBeamSearchDiversityRateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig26getBeamSearchDiversityRateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig26getBeamSearchDiversityRateEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getBeamSearchDiversityRateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1afdca37090c7283d286973e12af8d6c8b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBeamSearchDiversityRate</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig26getBeamSearchDiversityRateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getRepetitionPenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig20getRepetitionPenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig20getRepetitionPenaltyEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getRepetitionPenaltyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a4a39513b2dbeccdd039cec455a1916e3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRepetitionPenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getRepetitionPenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig18getPresencePenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig18getPresencePenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig18getPresencePenaltyEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getPresencePenaltyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af2ba08c30a1fceaa151bc3ce4e04dd19"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPresencePenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig18getPresencePenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig19getFrequencyPenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig19getFrequencyPenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig19getFrequencyPenaltyEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getFrequencyPenaltyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a106a21dae3dff8379ef1792a56fe46a6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFrequencyPenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig19getFrequencyPenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getPromptIgnoreLengthEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig21getPromptIgnoreLengthEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig21getPromptIgnoreLengthEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getPromptIgnoreLengthC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a5264385ea85957a61bd500ce843a3e25"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPromptIgnoreLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getPromptIgnoreLengthEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getLengthPenaltyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig16getLengthPenaltyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig16getLengthPenaltyEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getLengthPenaltyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a44fd8a1cc4a45d757aac154362118fbf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLengthPenalty</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getLengthPenaltyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getEarlyStoppingEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig16getEarlyStoppingEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig16getEarlyStoppingEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getEarlyStoppingC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ae8b64e9aecc63bf976ec9ae55e20f162"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEarlyStopping</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getEarlyStoppingEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getNoRepeatNgramSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig20getNoRepeatNgramSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig20getNoRepeatNgramSizeEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getNoRepeatNgramSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac3e9205236940fa410ba06df4e0ed4a8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNoRepeatNgramSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getNoRepeatNgramSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getNumReturnSequencesEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig21getNumReturnSequencesEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig21getNumReturnSequencesEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getNumReturnSequencesC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a8db12d54d1b1521a96b0c193051556f8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumReturnSequences</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getNumReturnSequencesEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getMinPEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig7getMinPEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig7getMinPEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getMinPC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aff8a03caff6476f3231b8f03f652804d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMinP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getMinPEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getBeamWidthArrayEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor14SamplingConfig17getBeamWidthArrayEv"></span><span id="_CPPv2NK12tensorrt_llm8executor14SamplingConfig17getBeamWidthArrayEv"></span><span id="tensorrt_llm::executor::SamplingConfig::getBeamWidthArrayC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a6570dffc50a012a4709c21a30b148cff"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getBeamWidthArray</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getBeamWidthArrayEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig12setBeamWidthE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig12setBeamWidthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig12setBeamWidthE10SizeType32"></span><span id="tensorrt_llm::executor::SamplingConfig::setBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a9d408d845ffd468e5c77a12644580acc"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBeamWidth</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setBeamWidthE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopKERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig7setTopKERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig7setTopKERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTopK__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a5461168ba842d21d25c4c584d9f9d023"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTopK</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topK</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopKERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopPERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig7setTopPERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig7setTopPERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTopP__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a913f54f64881543acd7709ed99da5085"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTopP</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topP</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopPERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig10setTopPMinERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig10setTopPMinERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig10setTopPMinERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTopPMin__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aa1789d40088cb15c5a07398607f2dd76"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTopPMin</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPMin</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10setTopPMinERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig15setTopPResetIdsERKNSt8optionalI11TokenIdTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig15setTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig15setTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTopPResetIds__std::optional:TokenIdType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab4926172b96ae86559d6247ba0b55b51"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTopPResetIds</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPResetIds</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15setTopPResetIdsERKNSt8optionalI11TokenIdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig12setTopPDecayERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig12setTopPDecayERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig12setTopPDecayERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTopPDecay__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab897fc960092c5e35a20e5eee4849d2d"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTopPDecay</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPDecay</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setTopPDecayERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig7setSeedERKNSt8optionalI14RandomSeedTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig7setSeedERKNSt8optionalI14RandomSeedTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig7setSeedERKNSt8optionalI14RandomSeedTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setSeed__std::optional:RandomSeedType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a28cf7200764ca1e5769e7d1ac4ca936c"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSeed</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE" title="tensorrt_llm::executor::RandomSeedType"><span class="n"><span class="pre">RandomSeedType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">seed</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setSeedERKNSt8optionalI14RandomSeedTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14setTemperatureERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14setTemperatureERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14setTemperatureERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setTemperature__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a22735fef3e4a93fb9d6f05e2405c8fc0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTemperature</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">temperature</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14setTemperatureERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig12setMinTokensERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig12setMinTokensERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig12setMinTokensERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setMinTokens__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ae64136223dada66024961ce6e689a069"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMinTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minTokens</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setMinTokensERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig26setBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig26setBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig26setBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setBeamSearchDiversityRate__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab22c4602e2c4ad41a8b5896d914d9a0c"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBeamSearchDiversityRate</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamSearchDiversityRate</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig26setBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig20setRepetitionPenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig20setRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig20setRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setRepetitionPenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac58bfcdc98186d18963554bb4b6c01f8"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRepetitionPenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">repetitionPenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20setRepetitionPenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig18setPresencePenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig18setPresencePenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig18setPresencePenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setPresencePenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1abe8dd822d08b0210e9f1fa3b36490fdf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPresencePenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">presencePenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18setPresencePenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig19setFrequencyPenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig19setFrequencyPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig19setFrequencyPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setFrequencyPenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a468c9e296264e676bd40c884c85bc4da"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFrequencyPenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">frequencyPenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19setFrequencyPenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig21setPromptIgnoreLengthERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig21setPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig21setPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setPromptIgnoreLength__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af02671dffd1c76ff98c762103cdc5610"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPromptIgnoreLength</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">promptIgnoreLength</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig21setPromptIgnoreLengthERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig16setLengthPenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig16setLengthPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig16setLengthPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setLengthPenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a9ea9013d86101dcdfb9d15a2b25998f5"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLengthPenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lengthPenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16setLengthPenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig16setEarlyStoppingERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig16setEarlyStoppingERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig16setEarlyStoppingERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setEarlyStopping__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a20097f9517e3f46e6584addf31a8fb3f"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEarlyStopping</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">earlyStopping</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16setEarlyStoppingERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig20setNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig20setNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig20setNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setNoRepeatNgramSize__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1add1f574a466f0c5a89933fe6f0d3dc13"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNoRepeatNgramSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">noRepeatNgramSize</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20setNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig21setNumReturnSequencesERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig21setNumReturnSequencesERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig21setNumReturnSequencesERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::setNumReturnSequences__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af3e76091583d194b540be92db684ea76"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumReturnSequences</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">numReturnSequences</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig21setNumReturnSequencesERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig7setMinPERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig7setMinPERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig7setMinPERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setMinP__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a9fbdc9261ce0ab587c2fa79d4eec2f38"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMinP</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minP</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setMinPERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig17setBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig17setBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig17setBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEE"></span><span id="tensorrt_llm::executor::SamplingConfig::setBeamWidthArray__std::optional:std::vector:SizeType32::CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ad28e065b63e885952e51f3aaacaeba53"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setBeamWidthArray</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamWidthArray</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17setBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig20updateNumReturnBeamsEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig20updateNumReturnBeamsEv"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig20updateNumReturnBeamsEv"></span><span id="tensorrt_llm::executor::SamplingConfig::updateNumReturnBeams"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af7693a504a1486108435bcb2cd8e523e"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">updateNumReturnBeams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20updateNumReturnBeamsEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig10mBeamWidthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig10mBeamWidthE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig10mBeamWidthE"></span><span id="tensorrt_llm::executor::SamplingConfig::mBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab75403888fa3c8d301912dbc350c35fb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBeamWidth</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mBeamWidthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The beam width. Default is 1 which disables beam search. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopKE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig5mTopKE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig5mTopKE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTopK__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac9850671f2d8eed17ecd6e8e4845f401"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTopK</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopKE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls number of logits to sample from. Default is 0 (all logits). </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig5mTopPE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig5mTopPE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTopP__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac442a2bcecf64be99729c882c9aaf09b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTopP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the top-P probability to sample from. Default is 0.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig8mTopPMinE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig8mTopPMinE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig8mTopPMinE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTopPMin__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac0692f29ae762728c4a7d1e438dad91e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTopPMin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig8mTopPMinE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls decay in the top-P algorithm. topPMin is lower-bound. Default is 1.e-6. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig13mTopPResetIdsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig13mTopPResetIdsE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig13mTopPResetIdsE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTopPResetIds__std::optional:TokenIdType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab0c410d2557fdf327ede1d67c292a7a2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTopPResetIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig13mTopPResetIdsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls decay in the top-P algorithm. Indicates where to reset the decay. Default is 1. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig10mTopPDecayE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig10mTopPDecayE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig10mTopPDecayE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTopPDecay__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af161a37e4aa10723d8ae5627943b4e03"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTopPDecay</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mTopPDecayE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls decay in the top-P algorithm. The decay value. Default is 1.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig5mSeedE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig5mSeedE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig5mSeedE"></span><span id="tensorrt_llm::executor::SamplingConfig::mSeed__std::optional:RandomSeedType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a3146e1b95e12aea945d14e0365bfe642"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE" title="tensorrt_llm::executor::RandomSeedType"><span class="n"><span class="pre">RandomSeedType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSeed</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mSeedE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the random seed used by the random number generator in sampling. Default is 0. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig12mTemperatureE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig12mTemperatureE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig12mTemperatureE"></span><span id="tensorrt_llm::executor::SamplingConfig::mTemperature__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ab5b9a3e150bdc536de54725c9210281e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTemperature</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12mTemperatureE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the modulation of logits when sampling new tokens. It can have values > 0.f. Default is 1.0f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig10mMinTokensE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig10mMinTokensE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig10mMinTokensE"></span><span id="tensorrt_llm::executor::SamplingConfig::mMinTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a1bd2df7e1d0d9e90e26126ab0e6f7632"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMinTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mMinTokensE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Lower bound on the number of tokens to generate. Values < 1 have no effect. Default is 1. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig24mBeamSearchDiversityRateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig24mBeamSearchDiversityRateE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig24mBeamSearchDiversityRateE"></span><span id="tensorrt_llm::executor::SamplingConfig::mBeamSearchDiversityRate__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a5d7fb8264ede8714d68ee323af7ce58f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBeamSearchDiversityRate</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig24mBeamSearchDiversityRateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the diversity in beam search. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig18mRepetitionPenaltyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig18mRepetitionPenaltyE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig18mRepetitionPenaltyE"></span><span id="tensorrt_llm::executor::SamplingConfig::mRepetitionPenalty__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1afc6184e856da9b06c54fd9093ca517bc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRepetitionPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18mRepetitionPenaltyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Used to penalize tokens based on how often they appear in the sequence. It can have any value > 0.f. Values < 1.f encourages repetition, values > 1.f discourages it. Default is 1.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig16mPresencePenaltyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig16mPresencePenaltyE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig16mPresencePenaltyE"></span><span id="tensorrt_llm::executor::SamplingConfig::mPresencePenalty__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1acee024920425ee593d16ac95113d5e2d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPresencePenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16mPresencePenaltyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Used to penalize tokens already present in the sequence (irrespective of the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig17mFrequencyPenaltyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig17mFrequencyPenaltyE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig17mFrequencyPenaltyE"></span><span id="tensorrt_llm::executor::SamplingConfig::mFrequencyPenalty__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a537a6f894e3c3093957c46774b23bb38"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFrequencyPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17mFrequencyPenaltyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Used to penalize tokens already present in the sequence (dependent on the number of appearances). It can have any values. Values < 0.f encourage repetition, values > 0.f discourage it. Default is 0.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig19mPromptIgnoreLengthE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig19mPromptIgnoreLengthE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig19mPromptIgnoreLengthE"></span><span id="tensorrt_llm::executor::SamplingConfig::mPromptIgnoreLength__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a6eb384ab3f1be49706fcef24483727a0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPromptIgnoreLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19mPromptIgnoreLengthE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls how many tokens to ignore from the prompt for presence and frequency penalties. Values <= 0 have no effect. Values > input (prompt) length will be clamped. Default is 0. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14mLengthPenaltyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14mLengthPenaltyE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14mLengthPenaltyE"></span><span id="tensorrt_llm::executor::SamplingConfig::mLengthPenalty__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a131e7f83f8b2a827761284f1bca2f934"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLengthPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14mLengthPenaltyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls how to penalize longer sequences in beam search. Default is 0.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14mEarlyStoppingE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14mEarlyStoppingE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14mEarlyStoppingE"></span><span id="tensorrt_llm::executor::SamplingConfig::mEarlyStopping__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a5bf3c3f8361b9ffd284f386ccd69eab0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEarlyStopping</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14mEarlyStoppingE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls whether the generation process finishes once beamWidth sentences are generated (ends with end_token). Default is 1. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig18mNoRepeatNgramSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig18mNoRepeatNgramSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig18mNoRepeatNgramSizeE"></span><span id="tensorrt_llm::executor::SamplingConfig::mNoRepeatNgramSize__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a782d671e6803a1cd7c3116004082b42e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNoRepeatNgramSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18mNoRepeatNgramSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls how many repeat ngram size are acceptable. Default is 1 << 30. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig19mNumReturnSequencesE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig19mNumReturnSequencesE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig19mNumReturnSequencesE"></span><span id="tensorrt_llm::executor::SamplingConfig::mNumReturnSequences__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aa36b13555a137de54e1a58106405e527"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumReturnSequences</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19mNumReturnSequencesE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The number of return sequences or beams. In beam search, the value should be less than or equal to mBeamWidth. In sampling, it specifies the total number of independently generated sequences. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig15mNumReturnBeamsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig15mNumReturnBeamsE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig15mNumReturnBeamsE"></span><span id="tensorrt_llm::executor::SamplingConfig::mNumReturnBeams__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aa9fd268711326300faebb1d64fc7aab0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumReturnBeams</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15mNumReturnBeamsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The number of beams to return. It is equal to beamWidth unless numReturnSequences is set. If beamWidth > 1 and numReturnSequences is set, then numReturnBeams is equal to numReturnSequences. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig5mMinPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig5mMinPE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig5mMinPE"></span><span id="tensorrt_llm::executor::SamplingConfig::mMinP__std::optional:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a88a357ebd9e4dae3983bd6420395717b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMinP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mMinPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the min_p scaling for sampling. It masks x which P_x < min_p * P_max, where P_x is probability of candidate x. Default is 0.f. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig15mBeamWidthArrayE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig15mBeamWidthArrayE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig15mBeamWidthArrayE"></span><span id="tensorrt_llm::executor::SamplingConfig::mBeamWidthArray__std::optional:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ada6901aa419abf8e9737eaf6d92aaade"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBeamWidthArray</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15mBeamWidthArrayE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Controls the beam width for each step for Variable-Beam-Width-Search. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkBeamWidthE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14checkBeamWidthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14checkBeamWidthE10SizeType32"></span><span id="tensorrt_llm::executor::SamplingConfig::checkBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a766b90d2d0f211808369d11906e561f6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">checkBeamWidth</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkBeamWidthE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopKERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig9checkTopKERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig9checkTopKERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTopK__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ac12283a0962b4087301acdb60aa7fef7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTopK</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topK</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopKERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopPERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig9checkTopPERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig9checkTopPERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTopP__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aa113c8eab8bf08c0824d6d836778ced7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTopP</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topP</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopPERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig12checkTopPMinERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig12checkTopPMinERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig12checkTopPMinERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTopPMin__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a82e79c31f7329090750a7612848a8a58"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTopPMin</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPMin</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12checkTopPMinERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig17checkTopPResetIdsERKNSt8optionalI11TokenIdTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig17checkTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig17checkTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTopPResetIds__std::optional:TokenIdType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1adf7a7cbc22eb3017045c09c3afebf54c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTopPResetIds</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE" title="tensorrt_llm::executor::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPResetIds</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17checkTopPResetIdsERKNSt8optionalI11TokenIdTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkTopPDecayERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14checkTopPDecayERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14checkTopPDecayERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTopPDecay__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a113ee3f518a17e0057edc1bdf6271c20"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTopPDecay</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">topPDecay</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkTopPDecayERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig16checkTemperatureERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig16checkTemperatureERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig16checkTemperatureERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkTemperature__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a182335063c59582c15b7c5b0a74db6cc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkTemperature</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">temperature</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16checkTemperatureERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkMinTokensERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig14checkMinTokensERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig14checkMinTokensERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkMinTokens__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af59789692b81d08bbdc9c905bc061efe"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkMinTokens</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minTokens</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkMinTokensERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig28checkBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig28checkBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig28checkBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkBeamSearchDiversityRate__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a49233a524c5e20983c880ca94f724e85"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkBeamSearchDiversityRate</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamSearchDiversityRate</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig28checkBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkRepetitionPenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig22checkRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig22checkRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkRepetitionPenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a7d60db03a7c3c300569f1e98d5879291"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkRepetitionPenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">repetitionpenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkRepetitionPenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkPromptIgnoreLengthERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig23checkPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig23checkPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkPromptIgnoreLength__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af4d9571628f7893108047fd5fd985580"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkPromptIgnoreLength</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">promptIgnoreLength</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkPromptIgnoreLengthERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkLengthPenaltyERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig18checkLengthPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig18checkLengthPenaltyERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkLengthPenalty__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a17345f9d99c28a24752041313dd82b45"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkLengthPenalty</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">lengthPenalty</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkLengthPenaltyERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkEarlyStoppingERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig18checkEarlyStoppingERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig18checkEarlyStoppingERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkEarlyStopping__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1aa7ef8e71751151a6dd61f39b9eb73c21"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkEarlyStopping</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">earlyStopping</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkEarlyStoppingERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig22checkNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig22checkNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkNoRepeatNgramSize__std::optional:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a9c0bf40fa2438f1bdbd2f815916c65bf"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkNoRepeatNgramSize</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">noRepeatNgramSize</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkNumReturnSequencesERKNSt8optionalI10SizeType32EE10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig23checkNumReturnSequencesERKNSt8optionalI10SizeType32EE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig23checkNumReturnSequencesERKNSt8optionalI10SizeType32EE10SizeType32"></span><span id="tensorrt_llm::executor::SamplingConfig::checkNumReturnSequences__std::optional:SizeType32:CR.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1a8ab5975ec921e4bf93a9a244509ca56a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkNumReturnSequences</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">numReturnSequences</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkNumReturnSequencesERKNSt8optionalI10SizeType32EE10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkMinPERKNSt8optionalI9FloatTypeEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig9checkMinPERKNSt8optionalI9FloatTypeEE"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig9checkMinPERKNSt8optionalI9FloatTypeEE"></span><span id="tensorrt_llm::executor::SamplingConfig::checkMinP__std::optional:FloatType:CR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1af48fc49ac54f25e195b4790776e1431c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">checkMinP</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE" title="tensorrt_llm::executor::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">minP</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkMinPERKNSt8optionalI9FloatTypeEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor14SamplingConfig19checkBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEEK10SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor14SamplingConfig19checkBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEEK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor14SamplingConfig19checkBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEEK10SizeType32"></span><span id="tensorrt_llm::executor::SamplingConfig::checkBeamWidthArray__std::optional:std::vector:SizeType32::CR.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SamplingConfig_1ae46765b0724489b336bc4604687c2698"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">checkBeamWidthArray</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">beamWidthArray</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19checkBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEEK10SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15SchedulerConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15SchedulerConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor15SchedulerConfigE"></span><span id="tensorrt_llm::executor::SchedulerConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SchedulerConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration class for the scheduler. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15SchedulerConfig15SchedulerConfigE23CapacitySchedulerPolicyNSt8optionalI21ContextChunkingPolicyEENSt8optionalI18DynamicBatchConfigEE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15SchedulerConfig15SchedulerConfigE23CapacitySchedulerPolicyNSt8optionalI21ContextChunkingPolicyEENSt8optionalI18DynamicBatchConfigEE"></span><span id="_CPPv2N12tensorrt_llm8executor15SchedulerConfig15SchedulerConfigE23CapacitySchedulerPolicyNSt8optionalI21ContextChunkingPolicyEENSt8optionalI18DynamicBatchConfigEE"></span><span id="tensorrt_llm::executor::SchedulerConfig::SchedulerConfig__CapacitySchedulerPolicy.std::optional:ContextChunkingPolicy:.std::optional:DynamicBatchConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1abe564b1c96406b37274f08781d30b1d1"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SchedulerConfig</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="tensorrt_llm::executor::CapacitySchedulerPolicy"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">capacitySchedulerPolicy</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="tensorrt_llm::executor::CapacitySchedulerPolicy"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE" title="tensorrt_llm::executor::CapacitySchedulerPolicy::kGUARANTEED_NO_EVICT"><span class="n"><span class="pre">kGUARANTEED_NO_EVICT</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE" title="tensorrt_llm::executor::ContextChunkingPolicy"><span class="n"><span class="pre">ContextChunkingPolicy</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">contextChunkingPolicy</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dynamicBatchConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig15SchedulerConfigE23CapacitySchedulerPolicyNSt8optionalI21ContextChunkingPolicyEENSt8optionalI18DynamicBatchConfigEE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15SchedulerConfigeqERK15SchedulerConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15SchedulerConfigeqERK15SchedulerConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor15SchedulerConfigeqERK15SchedulerConfig"></span><span id="tensorrt_llm::executor::SchedulerConfig::eq-operator__SchedulerConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1aa13c9b6ed3407575091225a06481b1a6"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE" title="tensorrt_llm::executor::SchedulerConfig"><span class="n"><span class="pre">SchedulerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfigeqERK15SchedulerConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15SchedulerConfig26getCapacitySchedulerPolicyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15SchedulerConfig26getCapacitySchedulerPolicyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15SchedulerConfig26getCapacitySchedulerPolicyEv"></span><span id="tensorrt_llm::executor::SchedulerConfig::getCapacitySchedulerPolicyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1aa72f57d07d885b6fd256189911076b8b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="tensorrt_llm::executor::CapacitySchedulerPolicy"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCapacitySchedulerPolicy</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig26getCapacitySchedulerPolicyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15SchedulerConfig24getContextChunkingPolicyEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15SchedulerConfig24getContextChunkingPolicyEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15SchedulerConfig24getContextChunkingPolicyEv"></span><span id="tensorrt_llm::executor::SchedulerConfig::getContextChunkingPolicyC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1a4e32082e39a6be5a295a0b334b2956d0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE" title="tensorrt_llm::executor::ContextChunkingPolicy"><span class="n"><span class="pre">ContextChunkingPolicy</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextChunkingPolicy</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig24getContextChunkingPolicyEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor15SchedulerConfig21getDynamicBatchConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor15SchedulerConfig21getDynamicBatchConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor15SchedulerConfig21getDynamicBatchConfigEv"></span><span id="tensorrt_llm::executor::SchedulerConfig::getDynamicBatchConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1a88b50045152cc75c4c036a989bded5bf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDynamicBatchConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig21getDynamicBatchConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15SchedulerConfig24mCapacitySchedulerPolicyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15SchedulerConfig24mCapacitySchedulerPolicyE"></span><span id="_CPPv2N12tensorrt_llm8executor15SchedulerConfig24mCapacitySchedulerPolicyE"></span><span id="tensorrt_llm::executor::SchedulerConfig::mCapacitySchedulerPolicy__CapacitySchedulerPolicy"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1a3c5dc1e328e369821e9b510bb89ddd0f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE" title="tensorrt_llm::executor::CapacitySchedulerPolicy"><span class="n"><span class="pre">CapacitySchedulerPolicy</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCapacitySchedulerPolicy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig24mCapacitySchedulerPolicyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The capacity scheduler policy. See CapacitySchedulerPolicy. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15SchedulerConfig22mContextChunkingPolicyE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15SchedulerConfig22mContextChunkingPolicyE"></span><span id="_CPPv2N12tensorrt_llm8executor15SchedulerConfig22mContextChunkingPolicyE"></span><span id="tensorrt_llm::executor::SchedulerConfig::mContextChunkingPolicy__std::optional:ContextChunkingPolicy:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1aa3f547ef1160624ae6e9ddce782967cf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE" title="tensorrt_llm::executor::ContextChunkingPolicy"><span class="n"><span class="pre">ContextChunkingPolicy</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mContextChunkingPolicy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig22mContextChunkingPolicyE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The context chunking policy. See ContextChunkingPolicy. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor15SchedulerConfig19mDynamicBatchConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor15SchedulerConfig19mDynamicBatchConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor15SchedulerConfig19mDynamicBatchConfigE"></span><span id="tensorrt_llm::executor::SchedulerConfig::mDynamicBatchConfig__std::optional:DynamicBatchConfig:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SchedulerConfig_1aaf3ec00bf67afe2ff3f0a7bb8268b6d8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE" title="tensorrt_llm::executor::DynamicBatchConfig"><span class="n"><span class="pre">DynamicBatchConfig</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDynamicBatchConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig19mDynamicBatchConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>The config for tuning batch size dynamically. See DynamicBatchSizeConfig. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25SpeculativeDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor25SpeculativeDecodingConfigE"></span><span id="tensorrt_llm::executor::SpeculativeDecodingConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SpeculativeDecodingConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Configuration for speculative decoding (both draft and target models) </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig25SpeculativeDecodingConfigEb">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25SpeculativeDecodingConfig25SpeculativeDecodingConfigEb"></span><span id="_CPPv2N12tensorrt_llm8executor25SpeculativeDecodingConfig25SpeculativeDecodingConfigEb"></span><span id="tensorrt_llm::executor::SpeculativeDecodingConfig::SpeculativeDecodingConfig__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SpeculativeDecodingConfig_1ac32ec832b367409979415f6e1bd7d7bb"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">fastLogits</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig25SpeculativeDecodingConfigEb" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor25SpeculativeDecodingConfigeqERK25SpeculativeDecodingConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor25SpeculativeDecodingConfigeqERK25SpeculativeDecodingConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor25SpeculativeDecodingConfigeqERK25SpeculativeDecodingConfig"></span><span id="tensorrt_llm::executor::SpeculativeDecodingConfig::eq-operator__SpeculativeDecodingConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SpeculativeDecodingConfig_1a1dab7496894ab51899cb8b809579eba0"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE" title="tensorrt_llm::executor::SpeculativeDecodingConfig"><span class="n"><span class="pre">SpeculativeDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor25SpeculativeDecodingConfigeqERK25SpeculativeDecodingConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig10fastLogitsE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor25SpeculativeDecodingConfig10fastLogitsE"></span><span id="_CPPv2N12tensorrt_llm8executor25SpeculativeDecodingConfig10fastLogitsE"></span><span id="tensorrt_llm::executor::SpeculativeDecodingConfig::fastLogits__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1SpeculativeDecodingConfig_1a669e6c71682aba0dcb1fb12856604b4d"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fastLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig10fastLogitsE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Send logits tensor directly from draft to target model. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE"></span><span id="_CPPv2N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE"></span><span id="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpeculativeDecodingFastLogitsInfo"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingFastLogitsInfo</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="docutils container">
|
||
<em>#include <executor.h></em></div>
|
||
<p>Struct that holds the logits information when using direct transfer. </p>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo8toTensorEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo8toTensorEv"></span><span id="_CPPv2NK12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo8toTensorEv"></span><span id="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo::toTensorC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpeculativeDecodingFastLogitsInfo_1ae4477369ea3c05b9e41148f94419fbde"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toTensor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo8toTensorEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Returns the struct serialized into a tensor that can be used as generation logits input. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo14draftRequestIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo14draftRequestIdE"></span><span id="_CPPv2N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo14draftRequestIdE"></span><span id="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo::draftRequestId__uint64_t"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpeculativeDecodingFastLogitsInfo_1ace8ba8a3ce4cf4586a7c48a95eeaf377"></span><span class="n"><span class="pre">uint64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftRequestId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo14draftRequestIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>Draft request id. </p>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo18draftParticipantIdE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo18draftParticipantIdE"></span><span id="_CPPv2N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo18draftParticipantIdE"></span><span id="tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo::draftParticipantId__int32_t"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1SpeculativeDecodingFastLogitsInfo_1a032fad3fc1dafa3f30ac89efad1fb484"></span><span class="n"><span class="pre">int32_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftParticipantId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo18draftParticipantIdE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p>MPI world rank of the draft model leader. </p>
|
||
</dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm3mpiE">
|
||
<span id="_CPPv3N12tensorrt_llm3mpiE"></span><span id="_CPPv2N12tensorrt_llm3mpiE"></span><span id="tensorrt_llm::mpi"></span><span class="target" id="namespacetensorrt__llm_1_1mpi"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mpi</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm3mpiE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="datatransceiverstate-h">
|
||
<h2>dataTransceiverState.h<a class="headerlink" href="#datatransceiverstate-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTransceiverState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEv"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEv"></span><span id="tensorrt_llm::executor::DataTransceiverState::DataTransceiverState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a874b210a70af5f39aa6d3ad291cfea92"></span><span class="sig-name descname"><span class="n"><span class="pre">DataTransceiverState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEN8kv_cache10CacheStateEN8kv_cache9CommStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEN8kv_cache10CacheStateEN8kv_cache9CommStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEN8kv_cache10CacheStateEN8kv_cache9CommStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState::DataTransceiverState__kv_cache::CacheState.kv_cache::CommState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1ab7149a7c168f5e19e100394b662521a9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTransceiverState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">cacheState</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">commState</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEN8kv_cache10CacheStateEN8kv_cache9CommStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState13setCacheStateEN8kv_cache10CacheStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState13setCacheStateEN8kv_cache10CacheStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState13setCacheStateEN8kv_cache10CacheStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState::setCacheState__kv_cache::CacheState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a7eadffedc76f4c8831733ef0ac3602c7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCacheState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState13setCacheStateEN8kv_cache10CacheStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20DataTransceiverState13getCacheStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20DataTransceiverState13getCacheStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20DataTransceiverState13getCacheStateEv"></span><span id="tensorrt_llm::executor::DataTransceiverState::getCacheStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a5f27f1431c6a8f5bc69bebcd27762b3a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getCacheState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState13getCacheStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState12setCommStateEN8kv_cache9CommStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState12setCommStateEN8kv_cache9CommStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState12setCommStateEN8kv_cache9CommStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState::setCommState__kv_cache::CommState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1aaea6c9b225a46322d9fea7c58761612d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setCommState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">state</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState12setCommStateEN8kv_cache9CommStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20DataTransceiverState12getCommStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20DataTransceiverState12getCommStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20DataTransceiverState12getCommStateEv"></span><span id="tensorrt_llm::executor::DataTransceiverState::getCommStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a2a110482ed371408c7d4e18efd085ccc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getCommState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState12getCommStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20DataTransceiverStateeqERK20DataTransceiverState">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20DataTransceiverStateeqERK20DataTransceiverState"></span><span id="_CPPv2NK12tensorrt_llm8executor20DataTransceiverStateeqERK20DataTransceiverState"></span><span id="tensorrt_llm::executor::DataTransceiverState::eq-operator__DataTransceiverStateCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a20f158f89e0ecbcb0715f2fba32a8b78"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE" title="tensorrt_llm::executor::DataTransceiverState"><span class="n"><span class="pre">DataTransceiverState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverStateeqERK20DataTransceiverState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor20DataTransceiverState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor20DataTransceiverState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor20DataTransceiverState8toStringEv"></span><span id="tensorrt_llm::executor::DataTransceiverState::toStringC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1a710cb31a3778dd18add8ef58ad2b91c3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState11mCacheStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState11mCacheStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState11mCacheStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState::mCacheState__std::optional:kv_cache::CacheState:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1ab4815ba252eadffff355b3d88f0b3009"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCacheState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState11mCacheStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor20DataTransceiverState10mCommStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor20DataTransceiverState10mCommStateE"></span><span id="_CPPv2N12tensorrt_llm8executor20DataTransceiverState10mCommStateE"></span><span id="tensorrt_llm::executor::DataTransceiverState::mCommState__std::optional:kv_cache::CommState:"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1DataTransceiverState_1ae789fca8b62cef084d597fc4ebb71340"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCommState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState10mCommStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor_1_1kv__cache"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kv_cache</span></span></span><br /></dt>
|
||
<dd><dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10AgentStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10AgentStateE"></span><span id="tensorrt_llm::executor::kv_cache::AgentState"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AgentState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateENSt6stringENSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateENSt6stringENSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateENSt6stringENSt6stringE"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::AgentState__ss.ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1a3811da59ccda26510ef568538f23ad8f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AgentState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">agentName</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">connectionInfo</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateENSt6stringENSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateEv"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::AgentState"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1a93c96b1f59e2493d5f52dbcd9943ca0e"></span><span class="sig-name descname"><span class="n"><span class="pre">AgentState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentStateeqERK10AgentState">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10AgentStateeqERK10AgentState"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10AgentStateeqERK10AgentState"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::eq-operator__AgentStateCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1ae4b36a422b23fa4c630a29ec3cf21896"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentStateeqERK10AgentState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10AgentState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10AgentState8toStringEv"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::toStringC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1a67247a110909a7d635c6e66e12692b1f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10mAgentNameE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10AgentState10mAgentNameE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10AgentState10mAgentNameE"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::mAgentName__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1a6c452cc2a53a6f569ac6faf6dd427e86"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAgentName</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10mAgentNameE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState15mConnectionInfoE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10AgentState15mConnectionInfoE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10AgentState15mConnectionInfoE"></span><span id="tensorrt_llm::executor::kv_cache::AgentState::mConnectionInfo__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1AgentState_1a50683a09065e14e0a198337d9d8c6a79"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mConnectionInfo</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState15mConnectionInfoE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheStateE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
|
||
<dl class="cpp enum-class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ad93549d3dc1c7eef825efe52974d1a04"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AttentionType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><p><em>Values:</em></p>
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ad93549d3dc1c7eef825efe52974d1a04a2eb05989f1bbfd98f356f4f3ac2ba2ec"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDEFAULT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp enumerator">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType4kMLAE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType4kMLAE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType4kMLAE"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ad93549d3dc1c7eef825efe52974d1a04a4fce02a8a7274ecfb0ebff8334abd92d"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType4kMLAE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE11ModelConfigRKN7runtime11WorldConfigERKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibb10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE11ModelConfigRKN7runtime11WorldConfigERKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibb10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE11ModelConfigRKN7runtime11WorldConfigERKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibb10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::CacheState__ModelConfig.runtime::WorldConfigCR.std::vector:SizeType32:CR.nvinfer1::DataType.AttentionType.i.b.b.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ac38ce17517a722c3b05bdf89989bdd53"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="runtime.html#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">attentionLayerNumPerPP</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="runtime.html#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType::kDEFAULT"><span class="n"><span class="pre">kDEFAULT</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvFactor</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBlockReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hasIndexerKCache</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerDimPerHead</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerKCacheQuantBlockSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">128</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE11ModelConfigRKN7runtime11WorldConfigERKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibb10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateENSt6vectorI10SizeType32EE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateENSt6vectorI10SizeType32EE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateENSt6vectorI10SizeType32EE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::CacheState__std::vector:SizeType32:.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.std::vector:SizeType32:CR.nvinfer1::DataType.AttentionType.i.b.i.i.b.b.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1aeed790fedb627464e2c7292dc5f8175e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">nbKvHeadPerLayer</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sizePerHead</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokensPerBlock</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensorParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">contextParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">attentionLayerNumPerPP</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="runtime.html#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType::kDEFAULT"><span class="n"><span class="pre">kDEFAULT</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvFactor</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableAttentionDP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">DPrank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">DPsize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBlockReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hasIndexerKCache</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerDimPerHead</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerKCacheQuantBlockSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">128</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateENSt6vectorI10SizeType32EE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::CacheState__SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.std::vector:SizeType32:CR.nvinfer1::DataType.AttentionType.i.b.i.i.b.b.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a1424cf07a4d052f390c23c329a57d740"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CacheState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbAttentionLayers</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbKvHeads</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sizePerHead</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokensPerBlock</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensorParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">contextParallelism</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">attentionLayerNumPerPP</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="runtime.html#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionType</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType::kDEFAULT"><span class="n"><span class="pre">kDEFAULT</span></span></a></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvFactor</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableAttentionDP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">DPrank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">DPsize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableBlockReuse</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">hasIndexerKCache</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerDimPerHead</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">indexerKCacheQuantBlockSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">128</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheStateeqERKN8kv_cache10CacheStateE">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheStateeqERKN8kv_cache10CacheStateE"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheStateeqERKN8kv_cache10CacheStateE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::eq-operator__kv_cache::CacheStateCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1af9854d1802f2c92abfc572e2b5273dd4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE" title="tensorrt_llm::executor::kv_cache"><span class="n"><span class="pre">kv_cache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE" title="tensorrt_llm::executor::kv_cache::CacheState"><span class="n"><span class="pre">CacheState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheStateeqERKN8kv_cache10CacheStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14getModelConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState14getModelConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState14getModelConfigEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getModelConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ab408625f42f4839f16577a3935dd379c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getModelConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14getModelConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState17getParallelConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState17getParallelConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState17getParallelConfigEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getParallelConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1aa78278db3c5db3c808530117744dc52f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getParallelConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState17getParallelConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState18getAttentionConfigEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState18getAttentionConfigEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState18getAttentionConfigEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getAttentionConfigC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1af2c1f22dd68e08d6a18879502f3bbfc0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig"><span class="n"><span class="pre">AttentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getAttentionConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState18getAttentionConfigEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11getDataTypeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState11getDataTypeEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getDataTypeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a76032092c3851b75234db0d3a8be1f51"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="runtime.html#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getEnableBlockReuseEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState19getEnableBlockReuseEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState19getEnableBlockReuseEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getEnableBlockReuseC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a3136275d758ca841aeff2bc28f9f9636"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEnableBlockReuse</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getEnableBlockReuseEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getHasIndexerKCacheEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState19getHasIndexerKCacheEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState19getHasIndexerKCacheEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getHasIndexerKCacheC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ab7d0a5aafdd0301542bc877cb9139354"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getHasIndexerKCache</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getHasIndexerKCacheEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState20getIndexerDimPerHeadEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState20getIndexerDimPerHeadEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState20getIndexerDimPerHeadEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getIndexerDimPerHeadC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a8937c6cc42be101f6b5a85970141b411"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIndexerDimPerHead</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState20getIndexerDimPerHeadEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState30getIndexerKCacheQuantBlockSizeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState30getIndexerKCacheQuantBlockSizeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState30getIndexerKCacheQuantBlockSizeEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::getIndexerKCacheQuantBlockSizeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a5be3e89906c712016a3d7dab5abb3843"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIndexerKCacheQuantBlockSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState30getIndexerKCacheQuantBlockSizeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState8toStringEv"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::toStringC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1aecf4552c456516093d4a34e2b72916ae"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState12mModelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState12mModelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState12mModelConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mModelConfig__ModelConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1aa8e5ce0c6f641b4c4e1eb4172e4f76b8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState12mModelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15mParallelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState15mParallelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState15mParallelConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mParallelConfig__ParallelConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a26fb69359fa22059813e2b2fbbbeea6a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mParallelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15mParallelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState9mDataTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState9mDataTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState9mDataTypeE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mDataType__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1af2d4f36b6124f636ce02b1406f7a7854"></span><a class="reference internal" href="runtime.html#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState9mDataTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState16mAttentionConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState16mAttentionConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState16mAttentionConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mAttentionConfig__AttentionConfig"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a4a44061c43c246ee59adecbb2219988f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig"><span class="n"><span class="pre">AttentionConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAttentionConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState16mAttentionConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mEnableBlockReuseE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState17mEnableBlockReuseE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState17mEnableBlockReuseE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mEnableBlockReuse__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a05c890463f61aa89b3bbcc7bc8949e5a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableBlockReuse</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mEnableBlockReuseE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mHasIndexerKCacheE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState17mHasIndexerKCacheE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState17mHasIndexerKCacheE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mHasIndexerKCache__b"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1ad7893d60bf06f405aeaf129c4b74a026"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mHasIndexerKCache</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mHasIndexerKCacheE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState18mIndexerDimPerHeadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState18mIndexerDimPerHeadE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState18mIndexerDimPerHeadE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mIndexerDimPerHead__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1af7096a38f86db54205890dc0bb6efea8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIndexerDimPerHead</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState18mIndexerDimPerHeadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState28mIndexerKCacheQuantBlockSizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState28mIndexerKCacheQuantBlockSizeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState28mIndexerKCacheQuantBlockSizeE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::mIndexerKCacheQuantBlockSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1a2d597f896e7b372dc697b84d9d70da3d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIndexerKCacheQuantBlockSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">128</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState28mIndexerKCacheQuantBlockSizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">tensorrt_llm::executor::Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1AttentionConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AttentionConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig15AttentionConfigE13AttentionTypei">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig15AttentionConfigE13AttentionTypei"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig15AttentionConfigE13AttentionTypei"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig::AttentionConfig__AttentionType.i"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1AttentionConfig_1a1631e42bdb0455b59db55567d9f84a59"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AttentionConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionType</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">kvFactor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig15AttentionConfigE13AttentionTypei" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigeqERK15AttentionConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigeqERK15AttentionConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigeqERK15AttentionConfig"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig::eq-operator__AttentionConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1AttentionConfig_1aa4a0721a432dc37451e3a1c8a9fdef9c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig"><span class="n"><span class="pre">AttentionConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigeqERK15AttentionConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig14mAttentionTypeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig14mAttentionTypeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig14mAttentionTypeE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig::mAttentionType__AttentionType"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1AttentionConfig_1afe8fc9545a02b0ae12cd376c9eb68cb8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE" title="tensorrt_llm::executor::kv_cache::CacheState::AttentionType"><span class="n"><span class="pre">AttentionType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAttentionType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig14mAttentionTypeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig9mKvFactorE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig9mKvFactorE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig9mKvFactorE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig::mKvFactor__i"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1AttentionConfig_1a92918abafde7ff0582a7f4a856b0a4f8"></span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mKvFactor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig9mKvFactorE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ModelConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigeqERK11ModelConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigeqERK11ModelConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigeqERK11ModelConfig"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig::eq-operator__ModelConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ModelConfig_1a219bc4ea7cb91fc8e8373acb37029561"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigeqERK11ModelConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig18mNbKvHeadsPerLayerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig18mNbKvHeadsPerLayerE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig18mNbKvHeadsPerLayerE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig::mNbKvHeadsPerLayer__std::vector:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ModelConfig_1ab06674d11814e671c661a68f47d264a0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNbKvHeadsPerLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig18mNbKvHeadsPerLayerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig12mSizePerHeadE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig12mSizePerHeadE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig12mSizePerHeadE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig::mSizePerHead__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ModelConfig_1ab77ef85183ef191ea1da15399f269560"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSizePerHead</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig12mSizePerHeadE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig15mTokensPerBlockE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig15mTokensPerBlockE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig15mTokensPerBlockE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ModelConfig::mTokensPerBlock__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ModelConfig_1a6ba0cfbe359d5ba11b78a10d3ccc309a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTokensPerBlock</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig15mTokensPerBlockE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ParallelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigeqERK14ParallelConfig">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigeqERK14ParallelConfig"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigeqERK14ParallelConfig"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::eq-operator__ParallelConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1a5545a2875c4f56a3a768057f6f46984e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE" title="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig"><span class="n"><span class="pre">ParallelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigeqERK14ParallelConfig" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mTensorParallelismE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mTensorParallelismE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mTensorParallelismE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mTensorParallelism__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1aab9ea317d531dd2b7565a4376954b8f1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTensorParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mTensorParallelismE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig20mPipelineParallelismE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig20mPipelineParallelismE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig20mPipelineParallelismE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mPipelineParallelism__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1afd0f8618c5e08e868d04daf15a1988b3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPipelineParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig20mPipelineParallelismE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig19mContextParallelismE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig19mContextParallelismE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig19mContextParallelismE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mContextParallelism__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1afe59a4e16f789181ae473aeb2c6031b2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mContextParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig19mContextParallelismE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mEnableAttentionDPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mEnableAttentionDPE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mEnableAttentionDPE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mEnableAttentionDP__b"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1a1a4091b782a2802f0b544df0754ac29c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableAttentionDP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mEnableAttentionDPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPrankE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPrankE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPrankE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mDPrank__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1ae965a841c8a6f0e42a15e6189026ae86"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDPrank</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPrankE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPsizeE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPsizeE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPsizeE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mDPsize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1ad992809a64081973bcf4d7a1ab435224"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDPsize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPsizeE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig23mAttentionLayerNumPerPPE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig23mAttentionLayerNumPerPPE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig23mAttentionLayerNumPerPPE"></span><span id="tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig::mAttentionLayerNumPerPP__std::vector:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1CacheState_1_1ParallelConfig_1a6bed17164e31ff0feaaba0bf5f3f3c68"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAttentionLayerNumPerPP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig23mAttentionLayerNumPerPPE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommStateE"></span><span id="tensorrt_llm::executor::kv_cache::CommState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateEv">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState9CommStateEv"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState9CommStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::CommState"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a22faf3dedde8d90a0b6c799705409c60"></span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10SizeType32EEi">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10SizeType32EEi"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10SizeType32EEi"></span><span id="tensorrt_llm::executor::kv_cache::CommState::CommState__std::vector:SizeType32:.i"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1ab188e69eb46d4938edb6588750e941fe"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ranks</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">selfIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10SizeType32EEi" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI11SocketStateEEi">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI11SocketStateEEi"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI11SocketStateEEi"></span><span id="tensorrt_llm::executor::kv_cache::CommState::CommState__std::vector:SocketState:.i"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a1b6b2467bd003fd265c303c30eaa0602"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">socketState</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">selfIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI11SocketStateEEi" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt8uint16_tENSt6stringE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt8uint16_tENSt6stringE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt8uint16_tENSt6stringE"></span><span id="tensorrt_llm::executor::kv_cache::CommState::CommState__std::uint16_t.ss"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a1d0d0238380c8e5fac3aa86ba42042b2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint16_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">port</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ip</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt8uint16_tENSt6stringE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10AgentStateEEi">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10AgentStateEEi"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10AgentStateEEi"></span><span id="tensorrt_llm::executor::kv_cache::CommState::CommState__std::vector:AgentState:.i"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1ae00620154dcc41776f8c9f32f071c86b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CommState</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="n sig-param"><span class="pre">agentState</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">selfIdx</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10AgentStateEEi" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10isMpiStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState10isMpiStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState10isMpiStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::isMpiStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1adf13114c0a7a8e9b4152b930a320575a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isMpiState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10isMpiStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13isSocketStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState13isSocketStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState13isSocketStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::isSocketStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1aa0b16fbb6bbea11cb489205c1b096293"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isSocketState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13isSocketStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState12isAgentStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState12isAgentStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState12isAgentStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::isAgentStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a9642d2551eac95665ca271c81d5369c3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isAgentState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState12isAgentStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState11getMpiStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState11getMpiStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState11getMpiStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::getMpiStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a227b9b9ab50d2c3dfde628f0fe038f32"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE" title="tensorrt_llm::executor::kv_cache::MpiState"><span class="n"><span class="pre">MpiState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getMpiState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState11getMpiStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState14getSocketStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState14getSocketStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState14getSocketStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::getSocketStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a36da005c9ce6ede8d38861a265dabc97"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSocketState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState14getSocketStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13getAgentStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState13getAgentStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState13getAgentStateEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::getAgentStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1af0d5eaf4f688a59cf1e1a443fc1cf4d6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getAgentState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13getAgentStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10getSelfIdxEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState10getSelfIdxEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState10getSelfIdxEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::getSelfIdxC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a7b7d55568bced2fe9449f9ea5320cdc0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSelfIdx</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10getSelfIdxEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommStateeqERK9CommState">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommStateeqERK9CommState"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommStateeqERK9CommState"></span><span id="tensorrt_llm::executor::kv_cache::CommState::eq-operator__CommStateCRC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a283d9cac18e2aba12cb7e0da03c1fbe4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommStateeqERK9CommState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache9CommState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache9CommState8toStringEv"></span><span id="tensorrt_llm::executor::kv_cache::CommState::toStringC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1acd1011b234682772824ff1d9dd868bc0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState6mStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState6mStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState6mStateE"></span><span id="tensorrt_llm::executor::kv_cache::CommState::mState__std::variant:std::monostate.MpiState.std::vector:SocketState:.std::vector:AgentState::"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a7e9796e81b194b769e04845efdcf2516"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">variant</span></span><span class="p"><span class="pre"><</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">monostate</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE" title="tensorrt_llm::executor::kv_cache::MpiState"><span class="n"><span class="pre">MpiState</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE" title="tensorrt_llm::executor::kv_cache::AgentState"><span class="n"><span class="pre">AgentState</span></span></a><span class="p"><span class="pre">></span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState6mStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache9CommState8mSelfIdxE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache9CommState8mSelfIdxE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache9CommState8mSelfIdxE"></span><span id="tensorrt_llm::executor::kv_cache::CommState::mSelfIdx__i"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1CommState_1a3dee19c3aba33f0e4e3c25a049a12851"></span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSelfIdx</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState8mSelfIdxE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
|
||
<dl>
|
||
<dt class="sig sig-object cpp">
|
||
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">tensorrt_llm::executor::Serialization</span></dt>
|
||
</dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8MpiStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8MpiStateE"></span><span id="tensorrt_llm::executor::kv_cache::MpiState"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1MpiState"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MpiState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiStateeqERK8MpiState">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache8MpiStateeqERK8MpiState"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache8MpiStateeqERK8MpiState"></span><span id="tensorrt_llm::executor::kv_cache::MpiState::eq-operator__MpiStateCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1MpiState_1a9add7106ff72b4527f909101394a58de"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE" title="tensorrt_llm::executor::kv_cache::MpiState"><span class="n"><span class="pre">MpiState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiStateeqERK8MpiState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache8MpiState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache8MpiState8toStringEv"></span><span id="tensorrt_llm::executor::kv_cache::MpiState::toStringC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1MpiState_1a08cca512e592c53b8a8b1b7fbf0f1fb0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache8MpiState6mRanksE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache8MpiState6mRanksE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache8MpiState6mRanksE"></span><span id="tensorrt_llm::executor::kv_cache::MpiState::mRanks__std::vector:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1MpiState_1a730c926ec3c87fa1aec4fa887ed20bf5"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E" title="tensorrt_llm::executor::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRanks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiState6mRanksE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11SocketStateE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11SocketStateE"></span><span id="tensorrt_llm::executor::kv_cache::SocketState"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1SocketState"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SocketState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketStateeqERK11SocketState">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache11SocketStateeqERK11SocketState"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache11SocketStateeqERK11SocketState"></span><span id="tensorrt_llm::executor::kv_cache::SocketState::eq-operator__SocketStateCRC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1SocketState_1a95165552bde02ceaae4736ef309d9601"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE" title="tensorrt_llm::executor::kv_cache::SocketState"><span class="n"><span class="pre">SocketState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketStateeqERK11SocketState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketState8toStringEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache11SocketState8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache11SocketState8toStringEv"></span><span id="tensorrt_llm::executor::kv_cache::SocketState::toStringC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1SocketState_1ae33603c80f7c7977ffefc3005520eb2c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketState8toStringEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState5mPortE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11SocketState5mPortE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11SocketState5mPortE"></span><span id="tensorrt_llm::executor::kv_cache::SocketState::mPort__std::uint16_t"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1SocketState_1a512fbb582759d66e8cd971c0aedd8b0a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint16_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPort</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState5mPortE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState3mIpE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11SocketState3mIpE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11SocketState3mIpE"></span><span id="tensorrt_llm::executor::kv_cache::SocketState::mIp__ss"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1SocketState_1ad8e62201a9a6d7077cb27e3058d08186"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIp</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState3mIpE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="cachecommunicator-h">
|
||
<h2>cacheCommunicator.h<a class="headerlink" href="#cachecommunicator-h" title="Link to this heading">#</a></h2>
|
||
<dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">executor</span></span></span><br /></dt>
|
||
<dd><dl class="cpp type">
|
||
<dt class="sig sig-object cpp">
|
||
<span class="target" id="namespacetensorrt__llm_1_1executor_1_1kv__cache"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kv_cache</span></span></span><br /></dt>
|
||
<dd><dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10ConnectionE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10ConnectionE"></span><span id="tensorrt_llm::executor::kv_cache::Connection"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1Connection"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Connection</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache10ConnectionD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache10ConnectionD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::Connection::~Connection"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1Connection_1ae10e91fbdbc387e2b0a1f449abdb6c0c"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~Connection</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4sendERK11DataContextPKv6size_t">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10Connection4sendERK11DataContextPKv6size_t"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10Connection4sendERK11DataContextPKv6size_t"></span><span id="tensorrt_llm::executor::kv_cache::Connection::send__DataContextCR.voidCP.sC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1Connection_1ae38e1609ad3f514e3132e5baee70b65e"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">send</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE" title="tensorrt_llm::executor::kv_cache::DataContext"><span class="n"><span class="pre">DataContext</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">ctx</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4sendERK11DataContextPKv6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4recvERK11DataContextPv6size_t">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10Connection4recvERK11DataContextPv6size_t"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10Connection4recvERK11DataContextPv6size_t"></span><span id="tensorrt_llm::executor::kv_cache::Connection::recv__DataContextCR.voidP.sC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1Connection_1ab07dbb048f817252e61043e6abbe4b87"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">recv</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE" title="tensorrt_llm::executor::kv_cache::DataContext"><span class="n"><span class="pre">DataContext</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">ctx</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4recvERK11DataContextPv6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection12isThreadSafeEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache10Connection12isThreadSafeEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache10Connection12isThreadSafeEv"></span><span id="tensorrt_llm::executor::kv_cache::Connection::isThreadSafeC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1Connection_1a31a082149469dc397e290fa3da0e7beb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isThreadSafe</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection12isThreadSafeEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp class">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17ConnectionManagerE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17ConnectionManagerE"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ConnectionManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerD0Ev">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17ConnectionManagerD0Ev"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17ConnectionManagerD0Ev"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::~ConnectionManager"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1a89d67f090d63d2bd147563b308197e82"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~ConnectionManager</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerD0Ev" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::recvConnect__DataContextCR.voidP.s"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1aba4c0592bbe30448a063f52a9c44a417"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionE" title="tensorrt_llm::executor::kv_cache::Connection"><span class="n"><span class="pre">Connection</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">recvConnect</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE" title="tensorrt_llm::executor::kv_cache::DataContext"><span class="n"><span class="pre">DataContext</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">ctx</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
|
||
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::getConnections__CommStateCR"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1a5b08a68b70fe12fcb4b9c760d121b03a"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre"><</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionE" title="tensorrt_llm::executor::kv_cache::Connection"><span class="n"><span class="pre">Connection</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">></span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getConnections</span></span></span><span class="sig-paren">(</span>
|
||
|
||
<dl>
|
||
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
|
||
</dl>
|
||
|
||
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"></span><span id="tensorrt_llm::executor::kv_cache::ConnectionManager::getCommStateC"></span><span class="target" id="classtensorrt__llm_1_1executor_1_1kv__cache_1_1ConnectionManager_1a1891e3f7d95d10d503768aa993b6debf"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE" title="tensorrt_llm::executor::kv_cache::CommState"><span class="n"><span class="pre">CommState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&</span></span><span class="sig-name descname"><span class="n"><span class="pre">getCommState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
<dl class="cpp struct">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11DataContextE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11DataContextE"></span><span id="tensorrt_llm::executor::kv_cache::DataContext"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1DataContext"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataContext</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE" title="Link to this definition">#</a><br /></dt>
|
||
<dd><div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext11DataContextEi">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11DataContext11DataContextEi"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11DataContext11DataContextEi"></span><span id="tensorrt_llm::executor::kv_cache::DataContext::DataContext__i"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1DataContext_1adabfaa156fdc10dbc52f525cd7223842"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataContext</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tag</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext11DataContextEi" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="cpp function">
|
||
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm8executor8kv_cache11DataContext6getTagEv">
|
||
<span id="_CPPv3NK12tensorrt_llm8executor8kv_cache11DataContext6getTagEv"></span><span id="_CPPv2NK12tensorrt_llm8executor8kv_cache11DataContext6getTagEv"></span><span id="tensorrt_llm::executor::kv_cache::DataContext::getTagC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1DataContext_1aed7e28a9ee248e7628ca208fce146239"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTag</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11DataContext6getTagEv" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
<div class="breathe-sectiondef docutils container">
|
||
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
|
||
<dl class="cpp var">
|
||
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext4mTagE">
|
||
<span id="_CPPv3N12tensorrt_llm8executor8kv_cache11DataContext4mTagE"></span><span id="_CPPv2N12tensorrt_llm8executor8kv_cache11DataContext4mTagE"></span><span id="tensorrt_llm::executor::kv_cache::DataContext::mTag__iC"></span><span class="target" id="structtensorrt__llm_1_1executor_1_1kv__cache_1_1DataContext_1ab7fc102718aaecd538065e5324637503"></span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTag</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext4mTagE" title="Link to this definition">#</a><br /></dt>
|
||
<dd></dd></dl>
|
||
|
||
</div>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
<dialog id="pst-secondary-sidebar-modal"></dialog>
|
||
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div
|
||
id="pst-page-navigation-heading-2"
|
||
class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> On this page
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#disaggserverutil-h">disaggServerUtil.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv412tensorrt_llm"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::disagg_executor</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::disagg_executor::DisaggExecutorOrchestrator</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator26DisaggExecutorOrchestratorERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorINSt10filesystem4pathEEERKNSt6vectorIN8executor14ExecutorConfigEEERKNSt6vectorIN8executor14ExecutorConfigEEEbb"><code class="docutils literal notranslate"><span class="pre">DisaggExecutorOrchestrator()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator14enqueueContextERKNSt6vectorIN5texec7RequestEEENSt8optionalIiEEb"><code class="docutils literal notranslate"><span class="pre">enqueueContext()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator17enqueueGenerationERKNSt6vectorIN5texec7RequestEEERKNSt6vectorI6IdTypeEENSt8optionalIiEEb"><code class="docutils literal notranslate"><span class="pre">enqueueGeneration()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator21awaitContextResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"><code class="docutils literal notranslate"><span class="pre">awaitContextResponses()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator24awaitGenerationResponsesERKNSt8optionalINSt6chrono12millisecondsEEENSt8optionalIiEE"><code class="docutils literal notranslate"><span class="pre">awaitGenerationResponses()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator10canEnqueueEv"><code class="docutils literal notranslate"><span class="pre">canEnqueue()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator19getContextExecutorsEv"><code class="docutils literal notranslate"><span class="pre">getContextExecutors()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator15getGenExecutorsEv"><code class="docutils literal notranslate"><span class="pre">getGenExecutors()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestratorD0Ev"><code class="docutils literal notranslate"><span class="pre">~DisaggExecutorOrchestrator()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor26DisaggExecutorOrchestrator5mImplE"><code class="docutils literal notranslate"><span class="pre">mImpl</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::disagg_executor::ResponseWithId</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERRN12tensorrt_llm8executor8ResponseE6IdType"><code class="docutils literal notranslate"><span class="pre">ResponseWithId()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERKN12tensorrt_llm8executor8ResponseE6IdType"><code class="docutils literal notranslate"><span class="pre">ResponseWithId()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERR14ResponseWithId"><code class="docutils literal notranslate"><span class="pre">ResponseWithId()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId14ResponseWithIdERK14ResponseWithId"><code class="docutils literal notranslate"><span class="pre">ResponseWithId()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERR14ResponseWithId"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdaSERK14ResponseWithId"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithIdD0Ev"><code class="docutils literal notranslate"><span class="pre">~ResponseWithId()</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId8responseE"><code class="docutils literal notranslate"><span class="pre">response</span></code></a></li>
|
||
<li class="toc-h7 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15disagg_executor14ResponseWithId3gidE"><code class="docutils literal notranslate"><span class="pre">gid</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#tensor-h">tensor.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5ShapeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Shape</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5Shape4BaseE"><code class="docutils literal notranslate"><span class="pre">Base</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5Shape9DimType64E"><code class="docutils literal notranslate"><span class="pre">DimType64</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeEv"><code class="docutils literal notranslate"><span class="pre">Shape()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeEPK9DimType64N4Base9size_typeE"><code class="docutils literal notranslate"><span class="pre">Shape()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor5Shape5ShapeENSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">Shape()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6TensorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Tensor</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">CudaStreamPtr</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor9copyToCpuEN6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">copyToCpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor12copyToPinnedEN6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">copyToPinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor18copyToPooledPinnedEN6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">copyToPooledPinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor13copyToManagedEN6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">copyToManaged()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor9copyToGpuEN6Tensor13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">copyToGpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorEv"><code class="docutils literal notranslate"><span class="pre">Tensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6TensorD0Ev"><code class="docutils literal notranslate"><span class="pre">~Tensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERK6Tensor"><code class="docutils literal notranslate"><span class="pre">Tensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorERR6Tensor"><code class="docutils literal notranslate"><span class="pre">Tensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6TensoraSERK6Tensor"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6TensoraSERR6Tensor"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor7getDataEv"><code class="docutils literal notranslate"><span class="pre">getData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor7getDataEv"><code class="docutils literal notranslate"><span class="pre">getData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor13getMemoryTypeEv"><code class="docutils literal notranslate"><span class="pre">getMemoryType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor8getShapeEv"><code class="docutils literal notranslate"><span class="pre">getShape()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor7getSizeEv"><code class="docutils literal notranslate"><span class="pre">getSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor14getSizeInBytesEv"><code class="docutils literal notranslate"><span class="pre">getSizeInBytes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor7setZeroE13CudaStreamPtr"><code class="docutils literal notranslate"><span class="pre">setZero()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor7setFromERK6Tensor13CudaStreamPtr"><code class="docutils literal notranslate"><span class="pre">setFrom()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6TensorcvbEv"><code class="docutils literal notranslate"><span class="pre">operator</span> <span class="pre">bool()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6TensoreqERK6Tensor"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6TensorneERK6Tensor"><code class="docutils literal notranslate"><span class="pre">operator!=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor3cpuE8DataType5Shape"><code class="docutils literal notranslate"><span class="pre">cpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor3cpuE6Tensor5Shape"><code class="docutils literal notranslate"><span class="pre">cpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6pinnedE8DataType5Shape"><code class="docutils literal notranslate"><span class="pre">pinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor6pinnedE6Tensor5Shape"><code class="docutils literal notranslate"><span class="pre">pinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor12pooledPinnedE8DataType5Shape"><code class="docutils literal notranslate"><span class="pre">pooledPinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor12pooledPinnedE6Tensor5Shape"><code class="docutils literal notranslate"><span class="pre">pooledPinned()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor7managedE8DataType5Shape"><code class="docutils literal notranslate"><span class="pre">managed()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor7managedE6Tensor5Shape"><code class="docutils literal notranslate"><span class="pre">managed()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor3gpuE8DataType13CudaStreamPtr5Shape"><code class="docutils literal notranslate"><span class="pre">gpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor3gpuE6Tensor13CudaStreamPtr5Shape"><code class="docutils literal notranslate"><span class="pre">gpu()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor2ofE8DataTypePv5Shape"><code class="docutils literal notranslate"><span class="pre">of()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorP1T5Shape"><code class="docutils literal notranslate"><span class="pre">of()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor2ofE6TensorR1T"><code class="docutils literal notranslate"><span class="pre">of()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor4ImplE"><code class="docutils literal notranslate"><span class="pre">Impl</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6TensorENSt10shared_ptrIN7runtime7ITensorEEE"><code class="docutils literal notranslate"><span class="pre">Tensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor6Tensor6copyToENSt10shared_ptrI4ImplEE13CudaStreamPtr"><code class="docutils literal notranslate"><span class="pre">copyTo()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor7mTensorE"><code class="docutils literal notranslate"><span class="pre">mTensor</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor6Tensor14getRuntimeTypeE8DataTypev"><code class="docutils literal notranslate"><span class="pre">getRuntimeType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6detail9toITensorERK6Tensor"><code class="docutils literal notranslate"><span class="pre">detail::toITensor</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Tensor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"><code class="docutils literal notranslate"><span class="pre">detail::ofITensor</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6detailE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::detail</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6detail9DimType64E"><code class="docutils literal notranslate"><span class="pre">DimType64</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6detail9toITensorERK6Tensor"><code class="docutils literal notranslate"><span class="pre">toITensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6detail9ofITensorENSt10shared_ptrIN7runtime7ITensorEEE"><code class="docutils literal notranslate"><span class="pre">ofITensor()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#transferagent-h">transferAgent.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cacheE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13TransferDescsE"><code class="docutils literal notranslate"><span class="pre">TransferDescs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache13RegisterDescsE"><code class="docutils literal notranslate"><span class="pre">RegisterDescs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SyncMessageE"><code class="docutils literal notranslate"><span class="pre">SyncMessage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache18ConnectionInfoTypeE"><code class="docutils literal notranslate"><span class="pre">ConnectionInfoType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryTypeE"><code class="docutils literal notranslate"><span class="pre">MemoryType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kDRAME"><code class="docutils literal notranslate"><span class="pre">kDRAM</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kVRAME"><code class="docutils literal notranslate"><span class="pre">kVRAM</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kBLKE"><code class="docutils literal notranslate"><span class="pre">kBLK</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType4kOBJE"><code class="docutils literal notranslate"><span class="pre">kOBJ</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryType5kFILEE"><code class="docutils literal notranslate"><span class="pre">kFILE</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOpE"><code class="docutils literal notranslate"><span class="pre">TransferOp</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp5kREADE"><code class="docutils literal notranslate"><span class="pre">kREAD</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10TransferOp6kWRITEE"><code class="docutils literal notranslate"><span class="pre">kWRITE</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeTransferAgentENSt10unique_ptrI17BaseTransferAgentEERKNSt6stringEDpRR4Args"><code class="docutils literal notranslate"><span class="pre">makeTransferAgent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IDpEN12tensorrt_llm8executor8kv_cache17makeLoopbackAgentENSt10shared_ptrI17BaseLoopbackAgentEERKNSt6stringEDpRR4Args"><code class="docutils literal notranslate"><span class="pre">makeLoopbackAgent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDescE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::AgentDesc</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc9AgentDescENSt6stringE"><code class="docutils literal notranslate"><span class="pre">AgentDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9AgentDesc19getBackendAgentDescEv"><code class="docutils literal notranslate"><span class="pre">getBackendAgentDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9AgentDesc17mBackendAgentDescE"><code class="docutils literal notranslate"><span class="pre">mBackendAgentDesc</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::BaseAgentConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig5mNameE"><code class="docutils literal notranslate"><span class="pre">mName</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig13useProgThreadE"><code class="docutils literal notranslate"><span class="pre">useProgThread</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15BaseAgentConfig11multiThreadE"><code class="docutils literal notranslate"><span class="pre">multiThread</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::BaseLoopbackAgent</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgentD0Ev"><code class="docutils literal notranslate"><span class="pre">~BaseLoopbackAgent()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseLoopbackAgent22executeLoopbackRequestERK11MemoryDescsRK9FileDescsb"><code class="docutils literal notranslate"><span class="pre">executeLoopbackRequest()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::BaseTransferAgent</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgentD0Ev"><code class="docutils literal notranslate"><span class="pre">~BaseTransferAgent()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent14registerMemoryERK13RegisterDescs"><code class="docutils literal notranslate"><span class="pre">registerMemory()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16deregisterMemoryERK13RegisterDescs"><code class="docutils literal notranslate"><span class="pre">deregisterMemory()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK9AgentDesc"><code class="docutils literal notranslate"><span class="pre">loadRemoteAgent()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent15loadRemoteAgentERKNSt6stringERK18ConnectionInfoType"><code class="docutils literal notranslate"><span class="pre">loadRemoteAgent()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent21invalidateRemoteAgentERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">invalidateRemoteAgent()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17getLocalAgentDescEv"><code class="docutils literal notranslate"><span class="pre">getLocalAgentDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22getLocalConnectionInfoEv"><code class="docutils literal notranslate"><span class="pre">getLocalConnectionInfo()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent22submitTransferRequestsERK15TransferRequest"><code class="docutils literal notranslate"><span class="pre">submitTransferRequests()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent17notifySyncMessageERKNSt6stringERK11SyncMessage"><code class="docutils literal notranslate"><span class="pre">notifySyncMessage()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent23getNotifiedSyncMessagesEv"><code class="docutils literal notranslate"><span class="pre">getNotifiedSyncMessages()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17BaseTransferAgent16checkRemoteDescsERKNSt6stringERK11MemoryDescs"><code class="docutils literal notranslate"><span class="pre">checkRemoteDescs()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::DynLibLoader</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9getHandleERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">getHandle()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor8kv_cache12DynLibLoader18getFunctionPointerE9FunctionTRKNSt6stringERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">getFunctionPointer()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderD0Ev"><code class="docutils literal notranslate"><span class="pre">~DynLibLoader()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderEv"><code class="docutils literal notranslate"><span class="pre">DynLibLoader()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader12DynLibLoaderERK12DynLibLoader"><code class="docutils literal notranslate"><span class="pre">DynLibLoader()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoaderaSERK12DynLibLoader"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader11getInstanceEv"><code class="docutils literal notranslate"><span class="pre">getInstance()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mDllMutexE"><code class="docutils literal notranslate"><span class="pre">mDllMutex</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader9mHandlersE"><code class="docutils literal notranslate"><span class="pre">mHandlers</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache12DynLibLoader5dlSymEPvPKc"><code class="docutils literal notranslate"><span class="pre">dlSym()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::FileDesc</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERKNSt6stringEi6mode_t6size_t"><code class="docutils literal notranslate"><span class="pre">FileDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERR8FileDesc"><code class="docutils literal notranslate"><span class="pre">FileDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERR8FileDesc"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescD0Ev"><code class="docutils literal notranslate"><span class="pre">~FileDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc5getFdEv"><code class="docutils literal notranslate"><span class="pre">getFd()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8FileDesc6getLenEv"><code class="docutils literal notranslate"><span class="pre">getLen()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc8FileDescERK8FileDesc"><code class="docutils literal notranslate"><span class="pre">FileDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDescaSERK8FileDesc"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc2fdE"><code class="docutils literal notranslate"><span class="pre">fd</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8FileDesc4mLenE"><code class="docutils literal notranslate"><span class="pre">mLen</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::FileDescs</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs9FileDescsERRNSt6vectorI8FileDescEE"><code class="docutils literal notranslate"><span class="pre">FileDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9FileDescs8getDescsEv"><code class="docutils literal notranslate"><span class="pre">getDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9FileDescs6mDescsE"><code class="docutils literal notranslate"><span class="pre">mDescs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDescE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::MemoryDesc</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescERKNSt6vectorIcEE8uint32_t"><code class="docutils literal notranslate"><span class="pre">MemoryDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescEPv6size_t8uint32_t"><code class="docutils literal notranslate"><span class="pre">MemoryDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc10MemoryDescE9uintptr_t6size_t8uint32_t"><code class="docutils literal notranslate"><span class="pre">MemoryDesc()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc7getAddrEv"><code class="docutils literal notranslate"><span class="pre">getAddr()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc6getLenEv"><code class="docutils literal notranslate"><span class="pre">getLen()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10MemoryDesc11getDeviceIdEv"><code class="docutils literal notranslate"><span class="pre">getDeviceId()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9serializeERK10MemoryDescRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc11deserializeERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserialize()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc14serializedSizeERK10MemoryDesc"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc5mAddrE"><code class="docutils literal notranslate"><span class="pre">mAddr</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc4mLenE"><code class="docutils literal notranslate"><span class="pre">mLen</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10MemoryDesc9mDeviceIdE"><code class="docutils literal notranslate"><span class="pre">mDeviceId</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::MemoryDescs</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs11MemoryDescsE10MemoryTypeNSt6vectorI10MemoryDescEE"><code class="docutils literal notranslate"><span class="pre">MemoryDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs7getTypeEv"><code class="docutils literal notranslate"><span class="pre">getType()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11MemoryDescs8getDescsEv"><code class="docutils literal notranslate"><span class="pre">getDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs5mTypeE"><code class="docutils literal notranslate"><span class="pre">mType</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11MemoryDescs6mDescsE"><code class="docutils literal notranslate"><span class="pre">mDescs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequestE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::TransferRequest</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest15TransferRequestE10TransferOp13TransferDescs13TransferDescsRKNSt6stringENSt8optionalI11SyncMessageEE"><code class="docutils literal notranslate"><span class="pre">TransferRequest()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest5getOpEv"><code class="docutils literal notranslate"><span class="pre">getOp()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getSrcDescsEv"><code class="docutils literal notranslate"><span class="pre">getSrcDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest11getDstDescsEv"><code class="docutils literal notranslate"><span class="pre">getDstDescs()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest13getRemoteNameEv"><code class="docutils literal notranslate"><span class="pre">getRemoteName()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache15TransferRequest14getSyncMessageEv"><code class="docutils literal notranslate"><span class="pre">getSyncMessage()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest3mOpE"><code class="docutils literal notranslate"><span class="pre">mOp</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mSrcDescsE"><code class="docutils literal notranslate"><span class="pre">mSrcDescs</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest9mDstDescsE"><code class="docutils literal notranslate"><span class="pre">mDstDescs</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest11mRemoteNameE"><code class="docutils literal notranslate"><span class="pre">mRemoteName</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache15TransferRequest12mSyncMessageE"><code class="docutils literal notranslate"><span class="pre">mSyncMessage</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::TransferStatus</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache14TransferStatusD0Ev"><code class="docutils literal notranslate"><span class="pre">~TransferStatus()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus11isCompletedEv"><code class="docutils literal notranslate"><span class="pre">isCompleted()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache14TransferStatus4waitEv"><code class="docutils literal notranslate"><span class="pre">wait()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#serialization-h">serialization.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13SerializationE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Serialization</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm13batch_manager16kv_cache_manager8BlockKeyERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization19deserializeBlockKeyERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeBlockKey()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeTimePointERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeTimePoint()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN18RequestPerfMetrics9TimePointERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN18RequestPerfMetrics9TimePointE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeRequestPerfMetricsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestPerfMetrics()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18RequestPerfMetricsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18RequestPerfMetrics"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeSamplingConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSamplingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14SamplingConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14SamplingConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeOutputConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeOutputConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12OutputConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12OutputConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeAdditionalModelOutputERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeAdditionalModelOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21AdditionalModelOutputRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21AdditionalModelOutput"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeExternalDraftTokensConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeExternalDraftTokensConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25ExternalDraftTokensConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25ExternalDraftTokensConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializePromptTuningConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializePromptTuningConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18PromptTuningConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18PromptTuningConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializeMultimodalInputERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeMultimodalInput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15MultimodalInputRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15MultimodalInput"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeMropeConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeMropeConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11MropeConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11MropeConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeLoraConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeLoraConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK10LoraConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK10LoraConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeCommStateERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeCommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache9CommStateERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache9CommStateE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeSocketStateERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSocketState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache11SocketStateERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache11SocketStateE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeAgentStateERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeAgentState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10AgentStateERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10AgentStateE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization21deserializeCacheStateERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeCacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN8kv_cache10CacheStateERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN8kv_cache10CacheStateE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDataTransceiverState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeDataTransceiverStateERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeDataTransceiverState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverStateRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20DataTransceiverState"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20DataTransceiverState"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeContextPhaseParamsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18ContextPhaseParamsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization18deserializeRequestERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeRequest()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK7RequestRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK7Request"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeTensorERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeTensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6TensorRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Tensor"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeSpecDecFastLogitsInfoERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSpecDecFastLogitsInfo()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK33SpeculativeDecodingFastLogitsInfoRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK33SpeculativeDecodingFastLogitsInfo"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeResultERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeResult()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK6ResultRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK6Result"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization27deserializeAdditionalOutputERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeAdditionalOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK16AdditionalOutputRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK16AdditionalOutput"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization19deserializeResponseERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeResponse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK8ResponseRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK8Response"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeResponsesERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeResponses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI8ResponseEE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKvCacheConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKvCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK13KvCacheConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK13KvCacheConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeDynamicBatchConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDynamicBatchConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18DynamicBatchConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18DynamicBatchConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializeSchedulerConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSchedulerConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15SchedulerConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15SchedulerConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization40deserializeExtendedRuntimePerfKnobConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeExtendedRuntimePerfKnobConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK29ExtendedRuntimePerfKnobConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK29ExtendedRuntimePerfKnobConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeParallelConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeParallelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ParallelConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ParallelConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization26deserializePeftCacheConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializePeftCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK15PeftCacheConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK15PeftCacheConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeOrchestratorConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeOrchestratorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18OrchestratorConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18OrchestratorConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeDecodingModeERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDecodingMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12DecodingModeRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12DecodingMode"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization34deserializeLookaheadDecodingConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeLookaheadDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK23LookaheadDecodingConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK23LookaheadDecodingConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeEagleConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeEagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11EagleConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11EagleConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeSpeculativeDecodingConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSpeculativeDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK25SpeculativeDecodingConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK25SpeculativeDecodingConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeGuidedDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization31deserializeGuidedDecodingParamsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeGuidedDecodingParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK20GuidedDecodingParamsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK20GuidedDecodingParams"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKvCacheRetentionConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKvCacheRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KvCacheRetentionConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KvCacheRetentionConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization36deserializeTokenRangeRetentionConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeTokenRangeRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN22KvCacheRetentionConfig25TokenRangeRetentionConfigE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeDecodingConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14DecodingConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14DecodingConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeDebugConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDebugConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK11DebugConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK11DebugConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeCacheTransceiverConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeCacheTransceiverConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22CacheTransceiverConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22CacheTransceiverConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeExecutorConfigERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeExecutorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14ExecutorConfigRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKvCacheStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKvCacheStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KvCacheStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KvCacheStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization30deserializeStaticBatchingStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeStaticBatchingStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK19StaticBatchingStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK19StaticBatchingStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization32deserializeInflightBatchingStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeInflightBatchingStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK21InflightBatchingStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK21InflightBatchingStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeSpecDecodingStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeSpecDecodingStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17SpecDecodingStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17SpecDecodingStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeIterationStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization25deserializeIterationStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeIterationStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK14IterationStats"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK14IterationStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI14IterationStatsEE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeIterationStatsVecERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeIterationStatsVec()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeDisServingRequestStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeDisServingRequestStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22DisServingRequestStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22DisServingRequestStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStageERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestStage()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStageRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStage"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeRequestStatsERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12RequestStatsRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12RequestStats"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestStatsPerIteration()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization35deserializeRequestStatsPerIterationERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestStatsPerIteration()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIterationRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK24RequestStatsPerIteration"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK24RequestStatsPerIteration"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt6vectorI24RequestStatsPerIterationEE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization38deserializeRequestStatsPerIterationVecERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeRequestStatsPerIterationVec()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKNSt5dequeI12KVCacheEventEE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization24deserializeKVCacheEventsERNSt6vectorIcEE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheEvents()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK12KVCacheEvent"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK12KVCacheEventRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization23deserializeKVCacheEventERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheEvent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheCreatedData"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheCreatedDataRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheCreatedDataERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheCreatedData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK17KVCacheStoredData"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK17KVCacheStoredDataRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization28deserializeKVCacheStoredDataERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheStoredData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK22KVCacheStoredBlockData"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK22KVCacheStoredBlockDataRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization33deserializeKVCacheStoredBlockDataERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheStoredBlockData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheRemovedData"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheRemovedDataRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheRemovedDataERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheRemovedData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization14serializedSizeE6size_tRK16KVCacheEventDiffI1TE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization9serializeEvRK16KVCacheEventDiffI1TERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor13Serialization27deserializeKVCacheEventDiffE16KVCacheEventDiffI1TERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheEventDiff()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERK18KVCacheUpdatedData"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERK18KVCacheUpdatedDataRNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization29deserializeKVCacheUpdatedDataERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeKVCacheUpdatedData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization14serializedSizeERKN12tensorrt_llm7runtime11UniqueTokenE"><code class="docutils literal notranslate"><span class="pre">serializedSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization9serializeERKN12tensorrt_llm7runtime11UniqueTokenERNSt7ostreamE"><code class="docutils literal notranslate"><span class="pre">serialize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization22deserializeUniqueTokenERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeUniqueToken()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization17deserializeStringERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization15deserializeBoolERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeBool()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13Serialization20deserializeModelTypeERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">deserializeModelType()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#types-h">types.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10SizeType64E"><code class="docutils literal notranslate"><span class="pre">SizeType64</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9FloatTypeE"><code class="docutils literal notranslate"><span class="pre">FloatType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11TokenIdTypeE"><code class="docutils literal notranslate"><span class="pre">TokenIdType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9VecTokensE"><code class="docutils literal notranslate"><span class="pre">VecTokens</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10BeamTokensE"><code class="docutils literal notranslate"><span class="pre">BeamTokens</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6IdTypeE"><code class="docutils literal notranslate"><span class="pre">IdType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16VecTokenExtraIdsE"><code class="docutils literal notranslate"><span class="pre">VecTokenExtraIds</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13IterationTypeE"><code class="docutils literal notranslate"><span class="pre">IterationType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14RandomSeedTypeE"><code class="docutils literal notranslate"><span class="pre">RandomSeedType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11VecLogProbsE"><code class="docutils literal notranslate"><span class="pre">VecLogProbs</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9StreamPtrE"><code class="docutils literal notranslate"><span class="pre">StreamPtr</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16MillisecondsTypeE"><code class="docutils literal notranslate"><span class="pre">MillisecondsType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15CacheSaltIDTypeE"><code class="docutils literal notranslate"><span class="pre">CacheSaltIDType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19LogitsPostProcessorE"><code class="docutils literal notranslate"><span class="pre">LogitsPostProcessor</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22LogitsPostProcessorMapE"><code class="docutils literal notranslate"><span class="pre">LogitsPostProcessorMap</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor26LogitsPostProcessorBatchedE"><code class="docutils literal notranslate"><span class="pre">LogitsPostProcessorBatched</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13MedusaChoicesE"><code class="docutils literal notranslate"><span class="pre">MedusaChoices</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12EagleChoicesE"><code class="docutils literal notranslate"><span class="pre">EagleChoices</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12PriorityTypeE"><code class="docutils literal notranslate"><span class="pre">PriorityType</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10BufferViewE"><code class="docutils literal notranslate"><span class="pre">BufferView</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataTypeE"><code class="docutils literal notranslate"><span class="pre">DataType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType5kBOOLE"><code class="docutils literal notranslate"><span class="pre">kBOOL</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType6kUINT8E"><code class="docutils literal notranslate"><span class="pre">kUINT8</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType5kINT8E"><code class="docutils literal notranslate"><span class="pre">kINT8</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT32E"><code class="docutils literal notranslate"><span class="pre">kINT32</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType6kINT64E"><code class="docutils literal notranslate"><span class="pre">kINT64</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType5kBF16E"><code class="docutils literal notranslate"><span class="pre">kBF16</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType4kFP8E"><code class="docutils literal notranslate"><span class="pre">kFP8</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP16E"><code class="docutils literal notranslate"><span class="pre">kFP16</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType5kFP32E"><code class="docutils literal notranslate"><span class="pre">kFP32</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8DataType8kUNKNOWNE"><code class="docutils literal notranslate"><span class="pre">kUNKNOWN</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11RequestTypeE"><code class="docutils literal notranslate"><span class="pre">RequestType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11RequestType35REQUEST_TYPE_CONTEXT_AND_GENERATIONE"><code class="docutils literal notranslate"><span class="pre">REQUEST_TYPE_CONTEXT_AND_GENERATION</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11RequestType25REQUEST_TYPE_CONTEXT_ONLYE"><code class="docutils literal notranslate"><span class="pre">REQUEST_TYPE_CONTEXT_ONLY</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11RequestType28REQUEST_TYPE_GENERATION_ONLYE"><code class="docutils literal notranslate"><span class="pre">REQUEST_TYPE_GENERATION_ONLY</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryTypeE"><code class="docutils literal notranslate"><span class="pre">MemoryType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kCPUE"><code class="docutils literal notranslate"><span class="pre">kCPU</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType11kCPU_PINNEDE"><code class="docutils literal notranslate"><span class="pre">kCPU_PINNED</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType15kCPU_PINNEDPOOLE"><code class="docutils literal notranslate"><span class="pre">kCPU_PINNEDPOOL</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kGPUE"><code class="docutils literal notranslate"><span class="pre">kGPU</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType4kUVME"><code class="docutils literal notranslate"><span class="pre">kUVM</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10MemoryType8kUNKNOWNE"><code class="docutils literal notranslate"><span class="pre">kUNKNOWN</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9ModelTypeE"><code class="docutils literal notranslate"><span class="pre">ModelType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9ModelType13kDECODER_ONLYE"><code class="docutils literal notranslate"><span class="pre">kDECODER_ONLY</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9ModelType13kENCODER_ONLYE"><code class="docutils literal notranslate"><span class="pre">kENCODER_ONLY</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor9ModelType16kENCODER_DECODERE"><code class="docutils literal notranslate"><span class="pre">kENCODER_DECODER</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12BatchingTypeE"><code class="docutils literal notranslate"><span class="pre">BatchingType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12BatchingType7kSTATICE"><code class="docutils literal notranslate"><span class="pre">kSTATIC</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12BatchingType9kINFLIGHTE"><code class="docutils literal notranslate"><span class="pre">kINFLIGHT</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicyE"><code class="docutils literal notranslate"><span class="pre">CapacitySchedulerPolicy</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy16kMAX_UTILIZATIONE"><code class="docutils literal notranslate"><span class="pre">kMAX_UTILIZATION</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy20kGUARANTEED_NO_EVICTE"><code class="docutils literal notranslate"><span class="pre">kGUARANTEED_NO_EVICT</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23CapacitySchedulerPolicy13kSTATIC_BATCHE"><code class="docutils literal notranslate"><span class="pre">kSTATIC_BATCH</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicyE"><code class="docutils literal notranslate"><span class="pre">ContextChunkingPolicy</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy24kFIRST_COME_FIRST_SERVEDE"><code class="docutils literal notranslate"><span class="pre">kFIRST_COME_FIRST_SERVED</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21ContextChunkingPolicy15kEQUAL_PROGRESSE"><code class="docutils literal notranslate"><span class="pre">kEQUAL_PROGRESS</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17CommunicationTypeE"><code class="docutils literal notranslate"><span class="pre">CommunicationType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17CommunicationType4kMPIE"><code class="docutils literal notranslate"><span class="pre">kMPI</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17CommunicationModeE"><code class="docutils literal notranslate"><span class="pre">CommunicationMode</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17CommunicationMode7kLEADERE"><code class="docutils literal notranslate"><span class="pre">kLEADER</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17CommunicationMode13kORCHESTRATORE"><code class="docutils literal notranslate"><span class="pre">kORCHESTRATOR</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStageE"><code class="docutils literal notranslate"><span class="pre">RequestStage</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStage7kQUEUEDE"><code class="docutils literal notranslate"><span class="pre">kQUEUED</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kENCODER_IN_PROGRESSE"><code class="docutils literal notranslate"><span class="pre">kENCODER_IN_PROGRESS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kCONTEXT_IN_PROGRESSE"><code class="docutils literal notranslate"><span class="pre">kCONTEXT_IN_PROGRESS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStage23kGENERATION_IN_PROGRESSE"><code class="docutils literal notranslate"><span class="pre">kGENERATION_IN_PROGRESS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStage20kGENERATION_COMPLETEE"><code class="docutils literal notranslate"><span class="pre">kGENERATION_COMPLETE</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReasonE"><code class="docutils literal notranslate"><span class="pre">FinishReason</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason13kNOT_FINISHEDE"><code class="docutils literal notranslate"><span class="pre">kNOT_FINISHED</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason7kEND_IDE"><code class="docutils literal notranslate"><span class="pre">kEND_ID</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason11kSTOP_WORDSE"><code class="docutils literal notranslate"><span class="pre">kSTOP_WORDS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason7kLENGTHE"><code class="docutils literal notranslate"><span class="pre">kLENGTH</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason10kTIMED_OUTE"><code class="docutils literal notranslate"><span class="pre">kTIMED_OUT</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12FinishReason10kCANCELLEDE"><code class="docutils literal notranslate"><span class="pre">kCANCELLED</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferModeE"><code class="docutils literal notranslate"><span class="pre">KvCacheTransferMode</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode4DRAME"><code class="docutils literal notranslate"><span class="pre">DRAM</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode3GDSE"><code class="docutils literal notranslate"><span class="pre">GDS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KvCacheTransferMode20POSIX_DEBUG_FALLBACKE"><code class="docutils literal notranslate"><span class="pre">POSIX_DEBUG_FALLBACK</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE23CapacitySchedulerPolicy"><code class="docutils literal notranslate"><span class="pre">operator<<()</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executorlsERNSt7ostreamE21ContextChunkingPolicy"><code class="docutils literal notranslate"><span class="pre">operator<<()</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIterationE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DebugTensorsPerIteration</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration4iterE"><code class="docutils literal notranslate"><span class="pre">iter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24DebugTensorsPerIteration12debugTensorsE"><code class="docutils literal notranslate"><span class="pre">debugTensors</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingModeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DecodingMode</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14UnderlyingTypeE"><code class="docutils literal notranslate"><span class="pre">UnderlyingType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode14useTemperatureEb"><code class="docutils literal notranslate"><span class="pre">useTemperature()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode22useOccurrencePenaltiesEb"><code class="docutils literal notranslate"><span class="pre">useOccurrencePenalties()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode18usePresencePenaltyEb"><code class="docutils literal notranslate"><span class="pre">usePresencePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20useRepetitionPenaltyEb"><code class="docutils literal notranslate"><span class="pre">useRepetitionPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19useFrequencyPenaltyEb"><code class="docutils literal notranslate"><span class="pre">useFrequencyPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useMinLengthEb"><code class="docutils literal notranslate"><span class="pre">useMinLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useBanTokensEb"><code class="docutils literal notranslate"><span class="pre">useBanTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode11useBanWordsEb"><code class="docutils literal notranslate"><span class="pre">useBanWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20useNoRepeatNgramSizeEb"><code class="docutils literal notranslate"><span class="pre">useNoRepeatNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12useStopWordsEb"><code class="docutils literal notranslate"><span class="pre">useStopWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode16useMaxLengthStopEb"><code class="docutils literal notranslate"><span class="pre">useMaxLengthStop()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode18useExplicitEosStopEb"><code class="docutils literal notranslate"><span class="pre">useExplicitEosStop()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode7useMinPEb"><code class="docutils literal notranslate"><span class="pre">useMinP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode26useVariableBeamWidthSearchEb"><code class="docutils literal notranslate"><span class="pre">useVariableBeamWidthSearch()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isAutoEv"><code class="docutils literal notranslate"><span class="pre">isAuto()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopKEv"><code class="docutils literal notranslate"><span class="pre">isTopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode6isTopPEv"><code class="docutils literal notranslate"><span class="pre">isTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isTopKorTopPEv"><code class="docutils literal notranslate"><span class="pre">isTopKorTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode13isTopKandTopPEv"><code class="docutils literal notranslate"><span class="pre">isTopKandTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isBeamSearchEv"><code class="docutils literal notranslate"><span class="pre">isBeamSearch()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode8isMedusaEv"><code class="docutils literal notranslate"><span class="pre">isMedusa()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode11isLookaheadEv"><code class="docutils literal notranslate"><span class="pre">isLookahead()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExplicitDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">isExplicitDraftTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isExternalDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">isExternalDraftTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode7isEagleEv"><code class="docutils literal notranslate"><span class="pre">isEagle()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode16isUseTemperatureEv"><code class="docutils literal notranslate"><span class="pre">isUseTemperature()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUsePresencePenaltyEv"><code class="docutils literal notranslate"><span class="pre">isUsePresencePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode21isUseFrequencyPenaltyEv"><code class="docutils literal notranslate"><span class="pre">isUseFrequencyPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseRepetitionPenaltyEv"><code class="docutils literal notranslate"><span class="pre">isUseRepetitionPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseMinLengthEv"><code class="docutils literal notranslate"><span class="pre">isUseMinLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseOccurrencePenaltyEv"><code class="docutils literal notranslate"><span class="pre">isUseOccurrencePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode12isUsePenaltyEv"><code class="docutils literal notranslate"><span class="pre">isUsePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode13isUseBanWordsEv"><code class="docutils literal notranslate"><span class="pre">isUseBanWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode22isUseNoRepeatNgramSizeEv"><code class="docutils literal notranslate"><span class="pre">isUseNoRepeatNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseBanTokensEv"><code class="docutils literal notranslate"><span class="pre">isUseBanTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode14isUseStopWordsEv"><code class="docutils literal notranslate"><span class="pre">isUseStopWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode18isUseMaxLengthStopEv"><code class="docutils literal notranslate"><span class="pre">isUseMaxLengthStop()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode20isUseExplicitEosStopEv"><code class="docutils literal notranslate"><span class="pre">isUseExplicitEosStop()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode17isUseStopCriteriaEv"><code class="docutils literal notranslate"><span class="pre">isUseStopCriteria()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9isUseMinPEv"><code class="docutils literal notranslate"><span class="pre">isUseMinP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode28isUseVariableBeamWidthSearchEv"><code class="docutils literal notranslate"><span class="pre">isUseVariableBeamWidthSearch()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingModeeqERK12DecodingMode"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12DecodingModeE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">DecodingMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode8getStateEv"><code class="docutils literal notranslate"><span class="pre">getState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode7getNameEv"><code class="docutils literal notranslate"><span class="pre">getName()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4AutoEv"><code class="docutils literal notranslate"><span class="pre">Auto()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4TopKEv"><code class="docutils literal notranslate"><span class="pre">TopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode4TopPEv"><code class="docutils literal notranslate"><span class="pre">TopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8TopKTopPEv"><code class="docutils literal notranslate"><span class="pre">TopKTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode10BeamSearchEv"><code class="docutils literal notranslate"><span class="pre">BeamSearch()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6MedusaEv"><code class="docutils literal notranslate"><span class="pre">Medusa()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9LookaheadEv"><code class="docutils literal notranslate"><span class="pre">Lookahead()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19ExplicitDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">ExplicitDraftTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19ExternalDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">ExternalDraftTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5EagleEv"><code class="docutils literal notranslate"><span class="pre">Eagle()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9anyBitSetE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">anyBitSet()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor12DecodingMode9allBitSetE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">allBitSet()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8setBitToE14UnderlyingTypeb"><code class="docutils literal notranslate"><span class="pre">setBitTo()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6mStateE"><code class="docutils literal notranslate"><span class="pre">mState</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kNumFlagsE"><code class="docutils literal notranslate"><span class="pre">kNumFlags</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseRepetitionPenaltiesE"><code class="docutils literal notranslate"><span class="pre">kUseRepetitionPenalties</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode22kUseFrequencyPenaltiesE"><code class="docutils literal notranslate"><span class="pre">kUseFrequencyPenalties</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUsePresencePenaltiesE"><code class="docutils literal notranslate"><span class="pre">kUsePresencePenalties</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode15kUseTemperatureE"><code class="docutils literal notranslate"><span class="pre">kUseTemperature</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseMinLengthE"><code class="docutils literal notranslate"><span class="pre">kUseMinLength</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode12kUseBanWordsE"><code class="docutils literal notranslate"><span class="pre">kUseBanWords</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseStopWordsE"><code class="docutils literal notranslate"><span class="pre">kUseStopWords</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode17kUseMaxLengthStopE"><code class="docutils literal notranslate"><span class="pre">kUseMaxLengthStop</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode19kUseExplicitEosStopE"><code class="docutils literal notranslate"><span class="pre">kUseExplicitEosStop</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode21kUseNoRepeatNgramSizeE"><code class="docutils literal notranslate"><span class="pre">kUseNoRepeatNgramSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode8kUseMinPE"><code class="docutils literal notranslate"><span class="pre">kUseMinP</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode27kUseVariableBeamWidthSearchE"><code class="docutils literal notranslate"><span class="pre">kUseVariableBeamWidthSearch</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode24kUseStandardStopCriteriaE"><code class="docutils literal notranslate"><span class="pre">kUseStandardStopCriteria</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode23kUseOccurrencePenaltiesE"><code class="docutils literal notranslate"><span class="pre">kUseOccurrencePenalties</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUsePenaltiesE"><code class="docutils literal notranslate"><span class="pre">kUsePenalties</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode13kUseBanTokensE"><code class="docutils literal notranslate"><span class="pre">kUseBanTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kAutoE"><code class="docutils literal notranslate"><span class="pre">kAuto</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopKE"><code class="docutils literal notranslate"><span class="pre">kTopK</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode5kTopPE"><code class="docutils literal notranslate"><span class="pre">kTopP</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode11kBeamSearchE"><code class="docutils literal notranslate"><span class="pre">kBeamSearch</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode7kMedusaE"><code class="docutils literal notranslate"><span class="pre">kMedusa</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode10kLookaheadE"><code class="docutils literal notranslate"><span class="pre">kLookahead</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20kExplicitDraftTokensE"><code class="docutils literal notranslate"><span class="pre">kExplicitDraftTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode20kExternalDraftTokensE"><code class="docutils literal notranslate"><span class="pre">kExternalDraftTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode6kEagleE"><code class="docutils literal notranslate"><span class="pre">kEagle</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12DecodingMode9kTopKTopPE"><code class="docutils literal notranslate"><span class="pre">kTopKTopP</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DisServingRequestStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStats17kvCacheTransferMSE"><code class="docutils literal notranslate"><span class="pre">kvCacheTransferMS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22DisServingRequestStats11kvCacheSizeE"><code class="docutils literal notranslate"><span class="pre">kvCacheSize</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::InflightBatchingStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats20numScheduledRequestsE"><code class="docutils literal notranslate"><span class="pre">numScheduledRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats18numContextRequestsE"><code class="docutils literal notranslate"><span class="pre">numContextRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats14numGenRequestsE"><code class="docutils literal notranslate"><span class="pre">numGenRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats17numPausedRequestsE"><code class="docutils literal notranslate"><span class="pre">numPausedRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12numCtxTokensE"><code class="docutils literal notranslate"><span class="pre">numCtxTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats12microBatchIdE"><code class="docutils literal notranslate"><span class="pre">microBatchId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21InflightBatchingStats26avgNumDecodedTokensPerIterE"><code class="docutils literal notranslate"><span class="pre">avgNumDecodedTokensPerIter</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::IterationStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats9timestampE"><code class="docutils literal notranslate"><span class="pre">timestamp</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats4iterE"><code class="docutils literal notranslate"><span class="pre">iter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats13iterLatencyMSE"><code class="docutils literal notranslate"><span class="pre">iterLatencyMS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats31newActiveRequestsQueueLatencyMSE"><code class="docutils literal notranslate"><span class="pre">newActiveRequestsQueueLatencyMS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20numNewActiveRequestsE"><code class="docutils literal notranslate"><span class="pre">numNewActiveRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17numActiveRequestsE"><code class="docutils literal notranslate"><span class="pre">numActiveRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17numQueuedRequestsE"><code class="docutils literal notranslate"><span class="pre">numQueuedRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20numCompletedRequestsE"><code class="docutils literal notranslate"><span class="pre">numCompletedRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats20maxNumActiveRequestsE"><code class="docutils literal notranslate"><span class="pre">maxNumActiveRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats18maxBatchSizeStaticE"><code class="docutils literal notranslate"><span class="pre">maxBatchSizeStatic</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats28maxBatchSizeTunerRecommendedE"><code class="docutils literal notranslate"><span class="pre">maxBatchSizeTunerRecommended</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19maxBatchSizeRuntimeE"><code class="docutils literal notranslate"><span class="pre">maxBatchSizeRuntime</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats18maxNumTokensStaticE"><code class="docutils literal notranslate"><span class="pre">maxNumTokensStatic</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats28maxNumTokensTunerRecommendedE"><code class="docutils literal notranslate"><span class="pre">maxNumTokensTunerRecommended</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19maxNumTokensRuntimeE"><code class="docutils literal notranslate"><span class="pre">maxNumTokensRuntime</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats11gpuMemUsageE"><code class="docutils literal notranslate"><span class="pre">gpuMemUsage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats11cpuMemUsageE"><code class="docutils literal notranslate"><span class="pre">cpuMemUsage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats14pinnedMemUsageE"><code class="docutils literal notranslate"><span class="pre">pinnedMemUsage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats12kvCacheStatsE"><code class="docutils literal notranslate"><span class="pre">kvCacheStats</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17crossKvCacheStatsE"><code class="docutils literal notranslate"><span class="pre">crossKvCacheStats</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats19staticBatchingStatsE"><code class="docutils literal notranslate"><span class="pre">staticBatchingStats</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats21inflightBatchingStatsE"><code class="docutils literal notranslate"><span class="pre">inflightBatchingStats</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14IterationStats17specDecodingStatsE"><code class="docutils literal notranslate"><span class="pre">specDecodingStats</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KvCacheStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12maxNumBlocksE"><code class="docutils literal notranslate"><span class="pre">maxNumBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats13freeNumBlocksE"><code class="docutils literal notranslate"><span class="pre">freeNumBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats13usedNumBlocksE"><code class="docutils literal notranslate"><span class="pre">usedNumBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats14tokensPerBlockE"><code class="docutils literal notranslate"><span class="pre">tokensPerBlock</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats16allocTotalBlocksE"><code class="docutils literal notranslate"><span class="pre">allocTotalBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats14allocNewBlocksE"><code class="docutils literal notranslate"><span class="pre">allocNewBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12reusedBlocksE"><code class="docutils literal notranslate"><span class="pre">reusedBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12missedBlocksE"><code class="docutils literal notranslate"><span class="pre">missedBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KvCacheStats12cacheHitRateE"><code class="docutils literal notranslate"><span class="pre">cacheHitRate</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetricsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestPerfMetrics</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9TimePointE"><code class="docutils literal notranslate"><span class="pre">TimePoint</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13timingMetricsE"><code class="docutils literal notranslate"><span class="pre">timingMetrics</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14kvCacheMetricsE"><code class="docutils literal notranslate"><span class="pre">kvCacheMetrics</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics19speculativeDecodingE"><code class="docutils literal notranslate"><span class="pre">speculativeDecoding</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics9firstIterE"><code class="docutils literal notranslate"><span class="pre">firstIter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics8lastIterE"><code class="docutils literal notranslate"><span class="pre">lastIter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics4iterE"><code class="docutils literal notranslate"><span class="pre">iter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetricsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestPerfMetrics::KvCacheMetrics</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics23numTotalAllocatedBlocksE"><code class="docutils literal notranslate"><span class="pre">numTotalAllocatedBlocks</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics21numNewAllocatedBlocksE"><code class="docutils literal notranslate"><span class="pre">numNewAllocatedBlocks</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numReusedBlocksE"><code class="docutils literal notranslate"><span class="pre">numReusedBlocks</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics15numMissedBlocksE"><code class="docutils literal notranslate"><span class="pre">numMissedBlocks</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics14KvCacheMetrics14kvCacheHitRateE"><code class="docutils literal notranslate"><span class="pre">kvCacheHitRate</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetricsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestPerfMetrics::SpeculativeDecodingMetrics</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics14acceptanceRateE"><code class="docutils literal notranslate"><span class="pre">acceptanceRate</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics24totalAcceptedDraftTokensE"><code class="docutils literal notranslate"><span class="pre">totalAcceptedDraftTokens</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics26SpeculativeDecodingMetrics16totalDraftTokensE"><code class="docutils literal notranslate"><span class="pre">totalDraftTokens</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetricsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestPerfMetrics::TimingMetrics</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11arrivalTimeE"><code class="docutils literal notranslate"><span class="pre">arrivalTime</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18firstScheduledTimeE"><code class="docutils literal notranslate"><span class="pre">firstScheduledTime</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics14firstTokenTimeE"><code class="docutils literal notranslate"><span class="pre">firstTokenTime</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics13lastTokenTimeE"><code class="docutils literal notranslate"><span class="pre">lastTokenTime</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics20kvCacheTransferStartE"><code class="docutils literal notranslate"><span class="pre">kvCacheTransferStart</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics18kvCacheTransferEndE"><code class="docutils literal notranslate"><span class="pre">kvCacheTransferEnd</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18RequestPerfMetrics13TimingMetrics11kvCacheSizeE"><code class="docutils literal notranslate"><span class="pre">kvCacheSize</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats2idE"><code class="docutils literal notranslate"><span class="pre">id</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats5stageE"><code class="docutils literal notranslate"><span class="pre">stage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22contextPrefillPositionE"><code class="docutils literal notranslate"><span class="pre">contextPrefillPosition</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats18numGeneratedTokensE"><code class="docutils literal notranslate"><span class="pre">numGeneratedTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats26avgNumDecodedTokensPerIterE"><code class="docutils literal notranslate"><span class="pre">avgNumDecodedTokensPerIter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats9scheduledE"><code class="docutils literal notranslate"><span class="pre">scheduled</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats6pausedE"><code class="docutils literal notranslate"><span class="pre">paused</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats15disServingStatsE"><code class="docutils literal notranslate"><span class="pre">disServingStats</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats26allocTotalBlocksPerRequestE"><code class="docutils literal notranslate"><span class="pre">allocTotalBlocksPerRequest</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats24allocNewBlocksPerRequestE"><code class="docutils literal notranslate"><span class="pre">allocNewBlocksPerRequest</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22reusedBlocksPerRequestE"><code class="docutils literal notranslate"><span class="pre">reusedBlocksPerRequest</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats22missedBlocksPerRequestE"><code class="docutils literal notranslate"><span class="pre">missedBlocksPerRequest</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12RequestStats24kvCacheHitRatePerRequestE"><code class="docutils literal notranslate"><span class="pre">kvCacheHitRatePerRequest</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIterationE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RequestStatsPerIteration</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration4iterE"><code class="docutils literal notranslate"><span class="pre">iter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor24RequestStatsPerIteration12requestStatsE"><code class="docutils literal notranslate"><span class="pre">requestStats</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::SpecDecodingStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats14numDraftTokensE"><code class="docutils literal notranslate"><span class="pre">numDraftTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats17numAcceptedTokensE"><code class="docutils literal notranslate"><span class="pre">numAcceptedTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats26numRequestsWithDraftTokensE"><code class="docutils literal notranslate"><span class="pre">numRequestsWithDraftTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats16acceptanceLengthE"><code class="docutils literal notranslate"><span class="pre">acceptanceLength</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13iterLatencyMSE"><code class="docutils literal notranslate"><span class="pre">iterLatencyMS</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17SpecDecodingStats13draftOverheadE"><code class="docutils literal notranslate"><span class="pre">draftOverhead</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStatsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::StaticBatchingStats</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats20numScheduledRequestsE"><code class="docutils literal notranslate"><span class="pre">numScheduledRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats18numContextRequestsE"><code class="docutils literal notranslate"><span class="pre">numContextRequests</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numCtxTokensE"><code class="docutils literal notranslate"><span class="pre">numCtxTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats12numGenTokensE"><code class="docutils literal notranslate"><span class="pre">numGenTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19StaticBatchingStats13emptyGenSlotsE"><code class="docutils literal notranslate"><span class="pre">emptyGenSlots</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_bEN12tensorrt_llm8executor10TypeTraitsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIbEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">bool</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIbE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsIfEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">float</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIfE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsI4halfEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">half</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsI4halfE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int32_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">std::int32_t</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int32_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7int64_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">std::int64_t</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7int64_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt6int8_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">std::int8_t</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt6int8_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">std::uint8_t</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsINSt7uint8_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor10TypeTraitsIP1TEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::TypeTraits<</span> <span class="pre">T</span> <span class="pre">*</span> <span class="pre">></span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10TypeTraitsIP1TE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#executor-h">executor.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm13batch_managerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::batch_manager</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm13batch_manager16kv_cache_managerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::batch_manager::kv_cache_manager</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17RetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">RetentionPriority</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDataE"><code class="docutils literal notranslate"><span class="pre">KVCacheEventData</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7versionEv"><code class="docutils literal notranslate"><span class="pre">version()</span></code></a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::AdditionalModelOutput</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput21AdditionalModelOutputENSt6stringEb"><code class="docutils literal notranslate"><span class="pre">AdditionalModelOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor21AdditionalModelOutputeqERK21AdditionalModelOutput"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor21AdditionalModelOutput13gatherContextE"><code class="docutils literal notranslate"><span class="pre">gatherContext</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::AdditionalOutput</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputENSt6stringE6Tensor"><code class="docutils literal notranslate"><span class="pre">AdditionalOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERK16AdditionalOutput"><code class="docutils literal notranslate"><span class="pre">AdditionalOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput16AdditionalOutputERR16AdditionalOutput"><code class="docutils literal notranslate"><span class="pre">AdditionalOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERK16AdditionalOutput"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputaSERR16AdditionalOutput"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutputD0Ev"><code class="docutils literal notranslate"><span class="pre">~AdditionalOutput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16AdditionalOutput6outputE"><code class="docutils literal notranslate"><span class="pre">output</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::CacheTransceiverConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendTypeE"><code class="docutils literal notranslate"><span class="pre">BackendType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType7DEFAULTE"><code class="docutils literal notranslate"><span class="pre">DEFAULT</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3MPIE"><code class="docutils literal notranslate"><span class="pre">MPI</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType3UCXE"><code class="docutils literal notranslate"><span class="pre">UCX</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig11BackendType4NIXLE"><code class="docutils literal notranslate"><span class="pre">NIXL</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22CacheTransceiverConfigENSt8optionalI11BackendTypeEENSt8optionalI6size_tEENSt8optionalIiEENSt8optionalIiEE"><code class="docutils literal notranslate"><span class="pre">CacheTransceiverConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfigeqERK22CacheTransceiverConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig14setBackendTypeENSt8optionalI11BackendTypeEE"><code class="docutils literal notranslate"><span class="pre">setBackendType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20setMaxTokensInBufferENSt8optionalI6size_tEE"><code class="docutils literal notranslate"><span class="pre">setMaxTokensInBuffer()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig22setKvTransferTimeoutMsENSt8optionalIiEE"><code class="docutils literal notranslate"><span class="pre">setKvTransferTimeoutMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig34setKvTransferSenderFutureTimeoutMsENSt8optionalIiEE"><code class="docutils literal notranslate"><span class="pre">setKvTransferSenderFutureTimeoutMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig20getMaxTokensInBufferEv"><code class="docutils literal notranslate"><span class="pre">getMaxTokensInBuffer()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig14getBackendTypeEv"><code class="docutils literal notranslate"><span class="pre">getBackendType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig22getKvTransferTimeoutMsEv"><code class="docutils literal notranslate"><span class="pre">getKvTransferTimeoutMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22CacheTransceiverConfig34getKvTransferSenderFutureTimeoutMsEv"><code class="docutils literal notranslate"><span class="pre">getKvTransferSenderFutureTimeoutMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig12mBackendTypeE"><code class="docutils literal notranslate"><span class="pre">mBackendType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig18mMaxTokensInBufferE"><code class="docutils literal notranslate"><span class="pre">mMaxTokensInBuffer</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig20mKvTransferTimeoutMsE"><code class="docutils literal notranslate"><span class="pre">mKvTransferTimeoutMs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22CacheTransceiverConfig32mKvTransferSenderFutureTimeoutMsE"><code class="docutils literal notranslate"><span class="pre">mKvTransferSenderFutureTimeoutMs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::ContextPhaseParams</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams13RequestIdTypeE"><code class="docutils literal notranslate"><span class="pre">RequestIdType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeNSt8optionalI9VecTokensEE"><code class="docutils literal notranslate"><span class="pre">ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypePvNSt8optionalI9VecTokensEE"><code class="docutils literal notranslate"><span class="pre">ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsE9VecTokens13RequestIdTypeRKNSt6vectorIcEENSt8optionalI9VecTokensEE"><code class="docutils literal notranslate"><span class="pre">ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERK18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams18ContextPhaseParamsERR18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERK18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsaSERR18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParamsD0Ev"><code class="docutils literal notranslate"><span class="pre">~ContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParamseqERK18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams17getFirstGenTokensEv"><code class="docutils literal notranslate"><span class="pre">getFirstGenTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NKR12tensorrt_llm8executor18ContextPhaseParams14getDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">getDraftTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NO12tensorrt_llm8executor18ContextPhaseParams17popFirstGenTokensEv"><code class="docutils literal notranslate"><span class="pre">popFirstGenTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getReqIdEv"><code class="docutils literal notranslate"><span class="pre">getReqId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams8getStateEv"><code class="docutils literal notranslate"><span class="pre">getState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8getStateEv"><code class="docutils literal notranslate"><span class="pre">getState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12releaseStateEv"><code class="docutils literal notranslate"><span class="pre">releaseState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18ContextPhaseParams18getSerializedStateEv"><code class="docutils literal notranslate"><span class="pre">getSerializedState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams8StatePtrE"><code class="docutils literal notranslate"><span class="pre">StatePtr</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mReqIdE"><code class="docutils literal notranslate"><span class="pre">mReqId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams15mFirstGenTokensE"><code class="docutils literal notranslate"><span class="pre">mFirstGenTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams6mStateE"><code class="docutils literal notranslate"><span class="pre">mState</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams12mDraftTokensE"><code class="docutils literal notranslate"><span class="pre">mDraftTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18ContextPhaseParams7deleterEPKv"><code class="docutils literal notranslate"><span class="pre">deleter()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DebugConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig11DebugConfigEbb9StringVec10SizeType32"><code class="docutils literal notranslate"><span class="pre">DebugConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfigeqERK11DebugConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig20getDebugInputTensorsEv"><code class="docutils literal notranslate"><span class="pre">getDebugInputTensors()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig21getDebugOutputTensorsEv"><code class="docutils literal notranslate"><span class="pre">getDebugOutputTensors()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig19getDebugTensorNamesEv"><code class="docutils literal notranslate"><span class="pre">getDebugTensorNames()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11DebugConfig28getDebugTensorsMaxIterationsEv"><code class="docutils literal notranslate"><span class="pre">getDebugTensorsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig20setDebugInputTensorsEb"><code class="docutils literal notranslate"><span class="pre">setDebugInputTensors()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig21setDebugOutputTensorsEb"><code class="docutils literal notranslate"><span class="pre">setDebugOutputTensors()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig19setDebugTensorNamesERK9StringVec"><code class="docutils literal notranslate"><span class="pre">setDebugTensorNames()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig28setDebugTensorsMaxIterationsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setDebugTensorsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig9StringVecE"><code class="docutils literal notranslate"><span class="pre">StringVec</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig18mDebugInputTensorsE"><code class="docutils literal notranslate"><span class="pre">mDebugInputTensors</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig19mDebugOutputTensorsE"><code class="docutils literal notranslate"><span class="pre">mDebugOutputTensors</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig17mDebugTensorNamesE"><code class="docutils literal notranslate"><span class="pre">mDebugTensorNames</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11DebugConfig26mDebugTensorsMaxIterationsE"><code class="docutils literal notranslate"><span class="pre">mDebugTensorsMaxIterations</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DecodingConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14DecodingConfigENSt8optionalI12DecodingModeEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI13MedusaChoicesEENSt8optionalI11EagleConfigEE"><code class="docutils literal notranslate"><span class="pre">DecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfigeqERK14DecodingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig15setDecodingModeERK12DecodingMode"><code class="docutils literal notranslate"><span class="pre">setDecodingMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig15getDecodingModeEv"><code class="docutils literal notranslate"><span class="pre">getDecodingMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig26setLookaheadDecodingConfigERK23LookaheadDecodingConfig"><code class="docutils literal notranslate"><span class="pre">setLookaheadDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig31enableSeamlessLookaheadDecodingEv"><code class="docutils literal notranslate"><span class="pre">enableSeamlessLookaheadDecoding()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig26getLookaheadDecodingConfigEv"><code class="docutils literal notranslate"><span class="pre">getLookaheadDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig33getLookaheadDecodingMaxNumRequestEv"><code class="docutils literal notranslate"><span class="pre">getLookaheadDecodingMaxNumRequest()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig16setMedusaChoicesERK13MedusaChoices"><code class="docutils literal notranslate"><span class="pre">setMedusaChoices()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig16getMedusaChoicesEv"><code class="docutils literal notranslate"><span class="pre">getMedusaChoices()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14setEagleConfigERK11EagleConfig"><code class="docutils literal notranslate"><span class="pre">setEagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14DecodingConfig14getEagleConfigEv"><code class="docutils literal notranslate"><span class="pre">getEagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig13mDecodingModeE"><code class="docutils literal notranslate"><span class="pre">mDecodingMode</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig24mLookaheadDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">mLookaheadDecodingConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig14mMedusaChoicesE"><code class="docutils literal notranslate"><span class="pre">mMedusaChoices</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig12mEagleConfigE"><code class="docutils literal notranslate"><span class="pre">mEagleConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14DecodingConfig31mLookaheadDecodingMaxNumRequestE"><code class="docutils literal notranslate"><span class="pre">mLookaheadDecodingMaxNumRequest</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DynamicBatchConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig18DynamicBatchConfigEbb10SizeType32NSt6vectorINSt4pairI10SizeType3210SizeType32EEEE"><code class="docutils literal notranslate"><span class="pre">DynamicBatchConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig34getDynamicBatchMovingAverageWindowEv"><code class="docutils literal notranslate"><span class="pre">getDynamicBatchMovingAverageWindow()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig24getEnableBatchSizeTuningEv"><code class="docutils literal notranslate"><span class="pre">getEnableBatchSizeTuning()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig27getEnableMaxNumTokensTuningEv"><code class="docutils literal notranslate"><span class="pre">getEnableMaxNumTokensTuning()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18DynamicBatchConfig17getBatchSizeTableEv"><code class="docutils literal notranslate"><span class="pre">getBatchSizeTable()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig39kDefaultDynamicBatchMovingAverageWindowE"><code class="docutils literal notranslate"><span class="pre">kDefaultDynamicBatchMovingAverageWindow</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22kDefaultBatchSizeTableE"><code class="docutils literal notranslate"><span class="pre">kDefaultBatchSizeTable</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig22mEnableBatchSizeTuningE"><code class="docutils literal notranslate"><span class="pre">mEnableBatchSizeTuning</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig25mEnableMaxNumTokensTuningE"><code class="docutils literal notranslate"><span class="pre">mEnableMaxNumTokensTuning</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig32mDynamicBatchMovingAverageWindowE"><code class="docutils literal notranslate"><span class="pre">mDynamicBatchMovingAverageWindow</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18DynamicBatchConfig15mBatchSizeTableE"><code class="docutils literal notranslate"><span class="pre">mBatchSizeTable</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::EagleConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig11EagleConfigENSt8optionalI12EagleChoicesEEbNSt8optionalIfEEbNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">EagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfigeqERK11EagleConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig15getEagleChoicesEv"><code class="docutils literal notranslate"><span class="pre">getEagleChoices()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig21getPosteriorThresholdEv"><code class="docutils literal notranslate"><span class="pre">getPosteriorThreshold()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig16isGreedySamplingEv"><code class="docutils literal notranslate"><span class="pre">isGreedySampling()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig14useDynamicTreeEv"><code class="docutils literal notranslate"><span class="pre">useDynamicTree()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11EagleConfig21getDynamicTreeMaxTopKEv"><code class="docutils literal notranslate"><span class="pre">getDynamicTreeMaxTopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19checkPosteriorValueERKNSt8optionalIfEE"><code class="docutils literal notranslate"><span class="pre">checkPosteriorValue()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig13mEagleChoicesE"><code class="docutils literal notranslate"><span class="pre">mEagleChoices</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig15mGreedySamplingE"><code class="docutils literal notranslate"><span class="pre">mGreedySampling</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19mPosteriorThresholdE"><code class="docutils literal notranslate"><span class="pre">mPosteriorThreshold</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig15mUseDynamicTreeE"><code class="docutils literal notranslate"><span class="pre">mUseDynamicTree</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11EagleConfig19mDynamicTreeMaxTopKE"><code class="docutils literal notranslate"><span class="pre">mDynamicTreeMaxTopK</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ExecutorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Executor</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERKNSt10filesystem4pathERKNSt10filesystem4pathE9ModelTypeRK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfigRKNSt8optionalINSt3mapINSt6stringE6TensorEEEE"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK10BufferViewRKNSt6stringERK10BufferViewRKNSt6stringE9ModelTypeRK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEERK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorENSt10shared_ptrI5ModelEENSt10shared_ptrI5ModelEERK14ExecutorConfig"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ExecutorD0Ev"><code class="docutils literal notranslate"><span class="pre">~Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERK8Executor"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ExecutoraSERK8Executor"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8ExecutorERR8Executor"><code class="docutils literal notranslate"><span class="pre">Executor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ExecutoraSERR8Executor"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor14enqueueRequestERK7Request"><code class="docutils literal notranslate"><span class="pre">enqueueRequest()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor15enqueueRequestsERKNSt6vectorI7RequestEE"><code class="docutils literal notranslate"><span class="pre">enqueueRequests()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">awaitResponses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERK6IdTypeRKNSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">awaitResponses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor14awaitResponsesERKNSt6vectorI6IdTypeEERKNSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">awaitResponses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Executor20getNumResponsesReadyERKNSt8optionalI6IdTypeEE"><code class="docutils literal notranslate"><span class="pre">getNumResponsesReady()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor13cancelRequestE6IdType"><code class="docutils literal notranslate"><span class="pre">cancelRequest()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor8shutdownEv"><code class="docutils literal notranslate"><span class="pre">shutdown()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor23getLatestIterationStatsEv"><code class="docutils literal notranslate"><span class="pre">getLatestIterationStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor21getLatestRequestStatsEv"><code class="docutils literal notranslate"><span class="pre">getLatestRequestStats()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor21getLatestDebugTensorsEv"><code class="docutils literal notranslate"><span class="pre">getLatestDebugTensors()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Executor18canEnqueueRequestsEv"><code class="docutils literal notranslate"><span class="pre">canEnqueueRequests()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Executor13isParticipantEv"><code class="docutils literal notranslate"><span class="pre">isParticipant()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Executor22getKVCacheEventManagerEv"><code class="docutils literal notranslate"><span class="pre">getKVCacheEventManager()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Executor5mImplE"><code class="docutils literal notranslate"><span class="pre">mImpl</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::ExecutorConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14ExecutorConfigE10SizeType3215SchedulerConfig13KvCacheConfigbb10SizeType3210SizeType3212BatchingTypeNSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI14ParallelConfigEERKNSt8optionalI15PeftCacheConfigEENSt8optionalI25LogitsPostProcessorConfigEENSt8optionalI14DecodingConfigEEbfNSt8optionalI10SizeType32EERK29ExtendedRuntimePerfKnobConfigNSt8optionalI11DebugConfigEE10SizeType328uint64_tNSt8optionalI25SpeculativeDecodingConfigEENSt8optionalI20GuidedDecodingConfigEENSt8optionalINSt6vectorI21AdditionalModelOutputEEEENSt8optionalI22CacheTransceiverConfigEEbbbb"><code class="docutils literal notranslate"><span class="pre">ExecutorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBeamWidthEv"><code class="docutils literal notranslate"><span class="pre">getMaxBeamWidth()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getSchedulerConfigEv"><code class="docutils literal notranslate"><span class="pre">getSchedulerConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getKvCacheConfigEv"><code class="docutils literal notranslate"><span class="pre">getKvCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21getSchedulerConfigRefEv"><code class="docutils literal notranslate"><span class="pre">getSchedulerConfigRef()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19getKvCacheConfigRefEv"><code class="docutils literal notranslate"><span class="pre">getKvCacheConfigRef()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getEnableChunkedContextEv"><code class="docutils literal notranslate"><span class="pre">getEnableChunkedContext()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getNormalizeLogProbsEv"><code class="docutils literal notranslate"><span class="pre">getNormalizeLogProbs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getIterStatsMaxIterationsEv"><code class="docutils literal notranslate"><span class="pre">getIterStatsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getRequestStatsMaxIterationsEv"><code class="docutils literal notranslate"><span class="pre">getRequestStatsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getBatchingTypeEv"><code class="docutils literal notranslate"><span class="pre">getBatchingType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxBatchSizeEv"><code class="docutils literal notranslate"><span class="pre">getMaxBatchSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxNumTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxNumTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getParallelConfigEv"><code class="docutils literal notranslate"><span class="pre">getParallelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig18getPeftCacheConfigEv"><code class="docutils literal notranslate"><span class="pre">getPeftCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig28getLogitsPostProcessorConfigEv"><code class="docutils literal notranslate"><span class="pre">getLogitsPostProcessorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig17getDecodingConfigEv"><code class="docutils literal notranslate"><span class="pre">getDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig22getUseGpuDirectStorageEv"><code class="docutils literal notranslate"><span class="pre">getUseGpuDirectStorage()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig20getGpuWeightsPercentEv"><code class="docutils literal notranslate"><span class="pre">getGpuWeightsPercent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig15getMaxQueueSizeEv"><code class="docutils literal notranslate"><span class="pre">getMaxQueueSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig32getExtendedRuntimePerfKnobConfigEv"><code class="docutils literal notranslate"><span class="pre">getExtendedRuntimePerfKnobConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig14getDebugConfigEv"><code class="docutils literal notranslate"><span class="pre">getDebugConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getRecvPollPeriodMsEv"><code class="docutils literal notranslate"><span class="pre">getRecvPollPeriodMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getMaxSeqIdleMicrosecondsEv"><code class="docutils literal notranslate"><span class="pre">getMaxSeqIdleMicroseconds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig16getSpecDecConfigEv"><code class="docutils literal notranslate"><span class="pre">getSpecDecConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig23getGuidedDecodingConfigEv"><code class="docutils literal notranslate"><span class="pre">getGuidedDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getAdditionalModelOutputsEv"><code class="docutils literal notranslate"><span class="pre">getAdditionalModelOutputs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getGatherGenerationLogitsEv"><code class="docutils literal notranslate"><span class="pre">getGatherGenerationLogits()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig24getPromptTableOffloadingEv"><code class="docutils literal notranslate"><span class="pre">getPromptTableOffloading()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig25getCacheTransceiverConfigEv"><code class="docutils literal notranslate"><span class="pre">getCacheTransceiverConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig19getEnableTrtOverlapEv"><code class="docutils literal notranslate"><span class="pre">getEnableTrtOverlap()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ExecutorConfig36getFailFastOnAttentionWindowTooLargeEv"><code class="docutils literal notranslate"><span class="pre">getFailFastOnAttentionWindowTooLarge()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBeamWidthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxBeamWidth()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxBatchSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxBatchSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxNumTokensE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxNumTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setSchedulerConfigERK15SchedulerConfig"><code class="docutils literal notranslate"><span class="pre">setSchedulerConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setKvCacheConfigERK13KvCacheConfig"><code class="docutils literal notranslate"><span class="pre">setKvCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setEnableChunkedContextEb"><code class="docutils literal notranslate"><span class="pre">setEnableChunkedContext()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setNormalizeLogProbsEb"><code class="docutils literal notranslate"><span class="pre">setNormalizeLogProbs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setIterStatsMaxIterationsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setIterStatsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setRequestStatsMaxIterationsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setRequestStatsMaxIterations()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setBatchingTypeE12BatchingType"><code class="docutils literal notranslate"><span class="pre">setBatchingType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setParallelConfigERK14ParallelConfig"><code class="docutils literal notranslate"><span class="pre">setParallelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18setPeftCacheConfigERK15PeftCacheConfig"><code class="docutils literal notranslate"><span class="pre">setPeftCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig28setLogitsPostProcessorConfigERK25LogitsPostProcessorConfig"><code class="docutils literal notranslate"><span class="pre">setLogitsPostProcessorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17setDecodingConfigERK14DecodingConfig"><code class="docutils literal notranslate"><span class="pre">setDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig22setUseGpuDirectStorageERKb"><code class="docutils literal notranslate"><span class="pre">setUseGpuDirectStorage()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20setGpuWeightsPercentERKf"><code class="docutils literal notranslate"><span class="pre">setGpuWeightsPercent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15setMaxQueueSizeERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setMaxQueueSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig32setExtendedRuntimePerfKnobConfigERK29ExtendedRuntimePerfKnobConfig"><code class="docutils literal notranslate"><span class="pre">setExtendedRuntimePerfKnobConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14setDebugConfigERK11DebugConfig"><code class="docutils literal notranslate"><span class="pre">setDebugConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setRecvPollPeriodMsERK10SizeType32"><code class="docutils literal notranslate"><span class="pre">setRecvPollPeriodMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setMaxSeqIdleMicrosecondsE8uint64_t"><code class="docutils literal notranslate"><span class="pre">setMaxSeqIdleMicroseconds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16setSpecDecConfigERK25SpeculativeDecodingConfig"><code class="docutils literal notranslate"><span class="pre">setSpecDecConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23setGuidedDecodingConfigERK20GuidedDecodingConfig"><code class="docutils literal notranslate"><span class="pre">setGuidedDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setAdditionalModelOutputsERKNSt6vectorI21AdditionalModelOutputEE"><code class="docutils literal notranslate"><span class="pre">setAdditionalModelOutputs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setGatherGenerationLogitsEb"><code class="docutils literal notranslate"><span class="pre">setGatherGenerationLogits()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig24setPromptTableOffloadingEb"><code class="docutils literal notranslate"><span class="pre">setPromptTableOffloading()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig25setCacheTransceiverConfigERK22CacheTransceiverConfig"><code class="docutils literal notranslate"><span class="pre">setCacheTransceiverConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig19setEnableTrtOverlapEb"><code class="docutils literal notranslate"><span class="pre">setEnableTrtOverlap()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig36setFailFastOnAttentionWindowTooLargeEb"><code class="docutils literal notranslate"><span class="pre">setFailFastOnAttentionWindowTooLarge()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultMaxSeqIdleMicrosecondsE"><code class="docutils literal notranslate"><span class="pre">kDefaultMaxSeqIdleMicroseconds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30kDefaultIterStatsMaxIterationsE"><code class="docutils literal notranslate"><span class="pre">kDefaultIterStatsMaxIterations</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig33kDefaultRequestStatsMaxIterationsE"><code class="docutils literal notranslate"><span class="pre">kDefaultRequestStatsMaxIterations</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBeamWidthE"><code class="docutils literal notranslate"><span class="pre">mMaxBeamWidth</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mSchedulerConfigE"><code class="docutils literal notranslate"><span class="pre">mSchedulerConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig14mKvCacheConfigE"><code class="docutils literal notranslate"><span class="pre">mKvCacheConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mEnableChunkedContextE"><code class="docutils literal notranslate"><span class="pre">mEnableChunkedContext</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mNormalizeLogProbsE"><code class="docutils literal notranslate"><span class="pre">mNormalizeLogProbs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mIterStatsMaxIterationsE"><code class="docutils literal notranslate"><span class="pre">mIterStatsMaxIterations</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mRequestStatsMaxIterationsE"><code class="docutils literal notranslate"><span class="pre">mRequestStatsMaxIterations</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mBatchingTypeE"><code class="docutils literal notranslate"><span class="pre">mBatchingType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxBatchSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxBatchSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxNumTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxNumTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mParallelConfigE"><code class="docutils literal notranslate"><span class="pre">mParallelConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig16mPeftCacheConfigE"><code class="docutils literal notranslate"><span class="pre">mPeftCacheConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mLogitsPostProcessorConfigE"><code class="docutils literal notranslate"><span class="pre">mLogitsPostProcessorConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig15mDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">mDecodingConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig20mUseGpuDirectStorageE"><code class="docutils literal notranslate"><span class="pre">mUseGpuDirectStorage</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig18mGpuWeightsPercentE"><code class="docutils literal notranslate"><span class="pre">mGpuWeightsPercent</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig13mMaxQueueSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxQueueSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig30mExtendedRuntimePerfKnobConfigE"><code class="docutils literal notranslate"><span class="pre">mExtendedRuntimePerfKnobConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig12mDebugConfigE"><code class="docutils literal notranslate"><span class="pre">mDebugConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mRecvPollPeriodMsE"><code class="docutils literal notranslate"><span class="pre">mRecvPollPeriodMs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mMaxSeqIdleMicrosecondsE"><code class="docutils literal notranslate"><span class="pre">mMaxSeqIdleMicroseconds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig26mSpeculativeDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">mSpeculativeDecodingConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig21mGuidedDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">mGuidedDecodingConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mAdditionalModelOutputsE"><code class="docutils literal notranslate"><span class="pre">mAdditionalModelOutputs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mCacheTransceiverConfigE"><code class="docutils literal notranslate"><span class="pre">mCacheTransceiverConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig23mGatherGenerationLogitsE"><code class="docutils literal notranslate"><span class="pre">mGatherGenerationLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig22mPromptTableOffloadingE"><code class="docutils literal notranslate"><span class="pre">mPromptTableOffloading</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig17mEnableTrtOverlapE"><code class="docutils literal notranslate"><span class="pre">mEnableTrtOverlap</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ExecutorConfig34mFailFastOnAttentionWindowTooLargeE"><code class="docutils literal notranslate"><span class="pre">mFailFastOnAttentionWindowTooLarge</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::ExtendedRuntimePerfKnobConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig29ExtendedRuntimePerfKnobConfigEbbb10SizeType32"><code class="docutils literal notranslate"><span class="pre">ExtendedRuntimePerfKnobConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfigeqERK29ExtendedRuntimePerfKnobConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17getMultiBlockModeEv"><code class="docutils literal notranslate"><span class="pre">getMultiBlockMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27getEnableContextFMHAFP32AccEv"><code class="docutils literal notranslate"><span class="pre">getEnableContextFMHAFP32Acc()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16getCudaGraphModeEv"><code class="docutils literal notranslate"><span class="pre">getCudaGraphMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21getCudaGraphCacheSizeEv"><code class="docutils literal notranslate"><span class="pre">getCudaGraphCacheSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig17setMultiBlockModeEb"><code class="docutils literal notranslate"><span class="pre">setMultiBlockMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig27setEnableContextFMHAFP32AccEb"><code class="docutils literal notranslate"><span class="pre">setEnableContextFMHAFP32Acc()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig16setCudaGraphModeEb"><code class="docutils literal notranslate"><span class="pre">setCudaGraphMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig21setCudaGraphCacheSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setCudaGraphCacheSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig15mMultiBlockModeE"><code class="docutils literal notranslate"><span class="pre">mMultiBlockMode</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig25mEnableContextFMHAFP32AccE"><code class="docutils literal notranslate"><span class="pre">mEnableContextFMHAFP32Acc</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig14mCudaGraphModeE"><code class="docutils literal notranslate"><span class="pre">mCudaGraphMode</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor29ExtendedRuntimePerfKnobConfig19mCudaGraphCacheSizeE"><code class="docutils literal notranslate"><span class="pre">mCudaGraphCacheSize</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::ExternalDraftTokensConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig25ExternalDraftTokensConfigE9VecTokensNSt8optionalI6TensorEERKNSt8optionalI9FloatTypeEERKNSt8optionalIbEE"><code class="docutils literal notranslate"><span class="pre">ExternalDraftTokensConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getTokensEv"><code class="docutils literal notranslate"><span class="pre">getTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig9getLogitsEv"><code class="docutils literal notranslate"><span class="pre">getLogits()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig22getAcceptanceThresholdEv"><code class="docutils literal notranslate"><span class="pre">getAcceptanceThreshold()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25ExternalDraftTokensConfig13getFastLogitsEv"><code class="docutils literal notranslate"><span class="pre">getFastLogits()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mTokensE"><code class="docutils literal notranslate"><span class="pre">mTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig7mLogitsE"><code class="docutils literal notranslate"><span class="pre">mLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig20mAcceptanceThresholdE"><code class="docutils literal notranslate"><span class="pre">mAcceptanceThreshold</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfig11mFastLogitsE"><code class="docutils literal notranslate"><span class="pre">mFastLogits</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::GuidedDecodingConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackendE"><code class="docutils literal notranslate"><span class="pre">GuidedDecodingBackend</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend9kXGRAMMARE"><code class="docutils literal notranslate"><span class="pre">kXGRAMMAR</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig21GuidedDecodingBackend11kLLGUIDANCEE"><code class="docutils literal notranslate"><span class="pre">kLLGUIDANCE</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig20GuidedDecodingConfigE21GuidedDecodingBackendNSt8optionalINSt6vectorINSt6stringEEEEENSt8optionalINSt6stringEEENSt8optionalINSt6vectorI11TokenIdTypeEEEE"><code class="docutils literal notranslate"><span class="pre">GuidedDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfigeqERK20GuidedDecodingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig10setBackendERK21GuidedDecodingBackend"><code class="docutils literal notranslate"><span class="pre">setBackend()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig10getBackendEv"><code class="docutils literal notranslate"><span class="pre">getBackend()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setEncodedVocabERKNSt6vectorINSt6stringEEE"><code class="docutils literal notranslate"><span class="pre">setEncodedVocab()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getEncodedVocabEv"><code class="docutils literal notranslate"><span class="pre">getEncodedVocab()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setTokenizerStrERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">setTokenizerStr()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getTokenizerStrEv"><code class="docutils literal notranslate"><span class="pre">getTokenizerStr()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig15setStopTokenIdsERKNSt6vectorI11TokenIdTypeEE"><code class="docutils literal notranslate"><span class="pre">setStopTokenIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig15getStopTokenIdsEv"><code class="docutils literal notranslate"><span class="pre">getStopTokenIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingConfig8validateEv"><code class="docutils literal notranslate"><span class="pre">validate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig8mBackendE"><code class="docutils literal notranslate"><span class="pre">mBackend</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mEncodedVocabE"><code class="docutils literal notranslate"><span class="pre">mEncodedVocab</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mTokenizerStrE"><code class="docutils literal notranslate"><span class="pre">mTokenizerStr</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingConfig13mStopTokenIdsE"><code class="docutils literal notranslate"><span class="pre">mStopTokenIds</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParamsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::GuidedDecodingParams</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideTypeE"><code class="docutils literal notranslate"><span class="pre">GuideType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType5kJSONE"><code class="docutils literal notranslate"><span class="pre">kJSON</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType12kJSON_SCHEMAE"><code class="docutils literal notranslate"><span class="pre">kJSON_SCHEMA</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType6kREGEXE"><code class="docutils literal notranslate"><span class="pre">kREGEX</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType13kEBNF_GRAMMARE"><code class="docutils literal notranslate"><span class="pre">kEBNF_GRAMMAR</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams9GuideType15kSTRUCTURAL_TAGE"><code class="docutils literal notranslate"><span class="pre">kSTRUCTURAL_TAG</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams20GuidedDecodingParamsE9GuideTypeNSt8optionalINSt6stringEEE"><code class="docutils literal notranslate"><span class="pre">GuidedDecodingParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParamseqERK20GuidedDecodingParams"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams12getGuideTypeEv"><code class="docutils literal notranslate"><span class="pre">getGuideType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20GuidedDecodingParams8getGuideEv"><code class="docutils literal notranslate"><span class="pre">getGuide()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams10mGuideTypeE"><code class="docutils literal notranslate"><span class="pre">mGuideType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20GuidedDecodingParams6mGuideE"><code class="docutils literal notranslate"><span class="pre">mGuide</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17JsonSerializationE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::JsonSerialization</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK14IterationStats"><code class="docutils literal notranslate"><span class="pre">toJsonStr()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK24RequestStatsPerIteration"><code class="docutils literal notranslate"><span class="pre">toJsonStr()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17JsonSerialization9toJsonStrERK12RequestStats"><code class="docutils literal notranslate"><span class="pre">toJsonStr()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KvCacheConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig13KvCacheConfigEbRKNSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI6size_tEEbRKNSt8optionalI9FloatTypeEENSt8optionalI17RetentionPriorityEE6size_tbbb10SizeType32RKNSt8optionalIN12tensorrt_llm7runtime15RuntimeDefaultsEEERK8uint64_t"><code class="docutils literal notranslate"><span class="pre">KvCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getEnableBlockReuseEv"><code class="docutils literal notranslate"><span class="pre">getEnableBlockReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEnablePartialReuseEv"><code class="docutils literal notranslate"><span class="pre">getEnablePartialReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getCopyOnPartialReuseEv"><code class="docutils literal notranslate"><span class="pre">getCopyOnPartialReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig12getMaxTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getMaxAttentionWindowVecEv"><code class="docutils literal notranslate"><span class="pre">getMaxAttentionWindowVec()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig18getSinkTokenLengthEv"><code class="docutils literal notranslate"><span class="pre">getSinkTokenLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig24getFreeGpuMemoryFractionEv"><code class="docutils literal notranslate"><span class="pre">getFreeGpuMemoryFraction()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig23getCrossKvCacheFractionEv"><code class="docutils literal notranslate"><span class="pre">getCrossKvCacheFraction()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getHostCacheSizeEv"><code class="docutils literal notranslate"><span class="pre">getHostCacheSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig16getOnboardBlocksEv"><code class="docutils literal notranslate"><span class="pre">getOnboardBlocks()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig30getSecondaryOffloadMinPriorityEv"><code class="docutils literal notranslate"><span class="pre">getSecondaryOffloadMinPriority()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig21getEventBufferMaxSizeEv"><code class="docutils literal notranslate"><span class="pre">getEventBufferMaxSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig9getUseUvmEv"><code class="docutils literal notranslate"><span class="pre">getUseUvm()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig34getAttentionDpEventsGatherPeriodMsEv"><code class="docutils literal notranslate"><span class="pre">getAttentionDpEventsGatherPeriodMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor13KvCacheConfig19getMaxGpuTotalBytesEv"><code class="docutils literal notranslate"><span class="pre">getMaxGpuTotalBytes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setEnableBlockReuseEb"><code class="docutils literal notranslate"><span class="pre">setEnableBlockReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEnablePartialReuseEb"><code class="docutils literal notranslate"><span class="pre">setEnablePartialReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setCopyOnPartialReuseEb"><code class="docutils literal notranslate"><span class="pre">setCopyOnPartialReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig12setMaxTokensENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setMaxTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setMaxAttentionWindowVecENSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setMaxAttentionWindowVec()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig18setSinkTokenLengthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setSinkTokenLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig24setFreeGpuMemoryFractionE9FloatType"><code class="docutils literal notranslate"><span class="pre">setFreeGpuMemoryFraction()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig23setCrossKvCacheFractionE9FloatType"><code class="docutils literal notranslate"><span class="pre">setCrossKvCacheFraction()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setHostCacheSizeE6size_t"><code class="docutils literal notranslate"><span class="pre">setHostCacheSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16setOnboardBlocksEb"><code class="docutils literal notranslate"><span class="pre">setOnboardBlocks()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig30setSecondaryOffloadMinPriorityENSt8optionalI17RetentionPriorityEE"><code class="docutils literal notranslate"><span class="pre">setSecondaryOffloadMinPriority()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21setEventBufferMaxSizeE6size_t"><code class="docutils literal notranslate"><span class="pre">setEventBufferMaxSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig9setUseUvmEb"><code class="docutils literal notranslate"><span class="pre">setUseUvm()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig34setAttentionDpEventsGatherPeriodMsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setAttentionDpEventsGatherPeriodMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19setMaxGpuTotalBytesE8uint64_t"><code class="docutils literal notranslate"><span class="pre">setMaxGpuTotalBytes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig34fillEmptyFieldsFromRuntimeDefaultsERKN12tensorrt_llm7runtime15RuntimeDefaultsE"><code class="docutils literal notranslate"><span class="pre">fillEmptyFieldsFromRuntimeDefaults()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22kDefaultGpuMemFractionE"><code class="docutils literal notranslate"><span class="pre">kDefaultGpuMemFraction</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mEnableBlockReuseE"><code class="docutils literal notranslate"><span class="pre">mEnableBlockReuse</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig10mMaxTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mMaxAttentionWindowVecE"><code class="docutils literal notranslate"><span class="pre">mMaxAttentionWindowVec</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig16mSinkTokenLengthE"><code class="docutils literal notranslate"><span class="pre">mSinkTokenLength</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig22mFreeGpuMemoryFractionE"><code class="docutils literal notranslate"><span class="pre">mFreeGpuMemoryFraction</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig21mCrossKvCacheFractionE"><code class="docutils literal notranslate"><span class="pre">mCrossKvCacheFraction</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mHostCacheSizeE"><code class="docutils literal notranslate"><span class="pre">mHostCacheSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig14mOnboardBlocksE"><code class="docutils literal notranslate"><span class="pre">mOnboardBlocks</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig28mSecondaryOffloadMinPriorityE"><code class="docutils literal notranslate"><span class="pre">mSecondaryOffloadMinPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEventBufferMaxSizeE"><code class="docutils literal notranslate"><span class="pre">mEventBufferMaxSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mEnablePartialReuseE"><code class="docutils literal notranslate"><span class="pre">mEnablePartialReuse</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig19mCopyOnPartialReuseE"><code class="docutils literal notranslate"><span class="pre">mCopyOnPartialReuse</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig7mUseUvmE"><code class="docutils literal notranslate"><span class="pre">mUseUvm</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig32mAttentionDpEventsGatherPeriodMsE"><code class="docutils literal notranslate"><span class="pre">mAttentionDpEventsGatherPeriodMs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor13KvCacheConfig17mMaxGpuTotalBytesE"><code class="docutils literal notranslate"><span class="pre">mMaxGpuTotalBytes</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheCreatedData</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheCreatedData22numBlocksPerCacheLevelE"><code class="docutils literal notranslate"><span class="pre">numBlocksPerCacheLevel</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEventE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheEvent</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent12KVCacheEventE6IdType16KVCacheEventData10SizeType32NSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">KVCacheEvent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent7eventIdE"><code class="docutils literal notranslate"><span class="pre">eventId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent4dataE"><code class="docutils literal notranslate"><span class="pre">data</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent10windowSizeE"><code class="docutils literal notranslate"><span class="pre">windowSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12KVCacheEvent15attentionDpRankE"><code class="docutils literal notranslate"><span class="pre">attentionDpRank</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm8executor16KVCacheEventDiffE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheEventDiff</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8oldValueE"><code class="docutils literal notranslate"><span class="pre">oldValue</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor16KVCacheEventDiff8newValueE"><code class="docutils literal notranslate"><span class="pre">newValue</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManagerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheEventManager</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager19KVCacheEventManagerENSt10shared_ptrIN12tensorrt_llm13batch_manager16kv_cache_manager18BaseKVCacheManagerEEE"><code class="docutils literal notranslate"><span class="pre">KVCacheEventManager()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager15getLatestEventsENSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">getLatestEvents()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor19KVCacheEventManager14kvCacheManagerE"><code class="docutils literal notranslate"><span class="pre">kvCacheManager</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheRemovedData</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheRemovedData11blockHashesE"><code class="docutils literal notranslate"><span class="pre">blockHashes</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KvCacheRetentionConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigEv"><code class="docutils literal notranslate"><span class="pre">KvCacheRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig22KvCacheRetentionConfigERKNSt6vectorI25TokenRangeRetentionConfigEE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE19KvCacheTransferModeRKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">KvCacheRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig29getTokenRangeRetentionConfigsEv"><code class="docutils literal notranslate"><span class="pre">getTokenRangeRetentionConfigs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig26getDecodeRetentionPriorityEv"><code class="docutils literal notranslate"><span class="pre">getDecodeRetentionPriority()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig19getDecodeDurationMsEv"><code class="docutils literal notranslate"><span class="pre">getDecodeDurationMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig15getTransferModeEv"><code class="docutils literal notranslate"><span class="pre">getTransferMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig12getDirectoryEv"><code class="docutils literal notranslate"><span class="pre">getDirectory()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig36getPerBlockRetentionPriorityDurationE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">getPerBlockRetentionPriorityDuration()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfigeqERK22KvCacheRetentionConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMinRetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">kMinRetentionPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig21kMaxRetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">kMaxRetentionPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25kDefaultRetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">kDefaultRetentionPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig27mTokenRangeRetentionConfigsE"><code class="docutils literal notranslate"><span class="pre">mTokenRangeRetentionConfigs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig24mDecodeRetentionPriorityE"><code class="docutils literal notranslate"><span class="pre">mDecodeRetentionPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig17mDecodeDurationMsE"><code class="docutils literal notranslate"><span class="pre">mDecodeDurationMs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig13mTransferModeE"><code class="docutils literal notranslate"><span class="pre">mTransferMode</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig10mDirectoryE"><code class="docutils literal notranslate"><span class="pre">mDirectory</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KvCacheRetentionConfig::TokenRangeRetentionConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig25TokenRangeRetentionConfigE10SizeType32NSt8optionalI10SizeType32EE17RetentionPriorityNSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">TokenRangeRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfigeqERK25TokenRangeRetentionConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10tokenStartE"><code class="docutils literal notranslate"><span class="pre">tokenStart</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8tokenEndE"><code class="docutils literal notranslate"><span class="pre">tokenEnd</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig8priorityE"><code class="docutils literal notranslate"><span class="pre">priority</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KvCacheRetentionConfig25TokenRangeRetentionConfig10durationMsE"><code class="docutils literal notranslate"><span class="pre">durationMs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheStoredBlockData</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData22KVCacheStoredBlockDataE6IdTypeN12tensorrt_llm7runtime15VecUniqueTokensENSt8optionalIN12tensorrt_llm7runtime14LoraTaskIdTypeEEE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">KVCacheStoredBlockData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData9blockHashE"><code class="docutils literal notranslate"><span class="pre">blockHash</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6tokensE"><code class="docutils literal notranslate"><span class="pre">tokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData6loraIdE"><code class="docutils literal notranslate"><span class="pre">loraId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData10cacheLevelE"><code class="docutils literal notranslate"><span class="pre">cacheLevel</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor22KVCacheStoredBlockData8priorityE"><code class="docutils literal notranslate"><span class="pre">priority</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheStoredData</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredData10parentHashE"><code class="docutils literal notranslate"><span class="pre">parentHash</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor17KVCacheStoredData6blocksE"><code class="docutils literal notranslate"><span class="pre">blocks</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedDataE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::KVCacheUpdatedData</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdType"><code class="docutils literal notranslate"><span class="pre">KVCacheUpdatedData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData18KVCacheUpdatedDataE6IdTypeNSt8optionalI16KVCacheEventDiffI10SizeType32EEENSt8optionalI16KVCacheEventDiffI10SizeType32EEE"><code class="docutils literal notranslate"><span class="pre">KVCacheUpdatedData()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData17cacheLevelUpdatedE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">cacheLevelUpdated()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData15priorityUpdatedE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">priorityUpdated()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData9blockHashE"><code class="docutils literal notranslate"><span class="pre">blockHash</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData10cacheLevelE"><code class="docutils literal notranslate"><span class="pre">cacheLevel</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18KVCacheUpdatedData8priorityE"><code class="docutils literal notranslate"><span class="pre">priority</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::LogitsPostProcessorConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig25LogitsPostProcessorConfigENSt8optionalI22LogitsPostProcessorMapEENSt8optionalI26LogitsPostProcessorBatchedEEb"><code class="docutils literal notranslate"><span class="pre">LogitsPostProcessorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig15getProcessorMapEv"><code class="docutils literal notranslate"><span class="pre">getProcessorMap()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig19getProcessorBatchedEv"><code class="docutils literal notranslate"><span class="pre">getProcessorBatched()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25LogitsPostProcessorConfig12getReplicateEv"><code class="docutils literal notranslate"><span class="pre">getReplicate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig15setProcessorMapERK22LogitsPostProcessorMap"><code class="docutils literal notranslate"><span class="pre">setProcessorMap()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig19setProcessorBatchedERK26LogitsPostProcessorBatched"><code class="docutils literal notranslate"><span class="pre">setProcessorBatched()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig12setReplicateEb"><code class="docutils literal notranslate"><span class="pre">setReplicate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig13mProcessorMapE"><code class="docutils literal notranslate"><span class="pre">mProcessorMap</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig17mProcessorBatchedE"><code class="docutils literal notranslate"><span class="pre">mProcessorBatched</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25LogitsPostProcessorConfig10mReplicateE"><code class="docutils literal notranslate"><span class="pre">mReplicate</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::LookaheadDecodingConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">LookaheadDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig23LookaheadDecodingConfigEv"><code class="docutils literal notranslate"><span class="pre">LookaheadDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfigeqERK23LookaheadDecodingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig3getEv"><code class="docutils literal notranslate"><span class="pre">get()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig13getWindowSizeEv"><code class="docutils literal notranslate"><span class="pre">getWindowSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig12getNgramSizeEv"><code class="docutils literal notranslate"><span class="pre">getNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig22getVerificationSetSizeEv"><code class="docutils literal notranslate"><span class="pre">getVerificationSetSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig28calculateSpeculativeResourceEv"><code class="docutils literal notranslate"><span class="pre">calculateSpeculativeResource()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor23LookaheadDecodingConfig4isLEERK23LookaheadDecodingConfig"><code class="docutils literal notranslate"><span class="pre">isLE()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig33calculateSpeculativeResourceTupleE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">calculateSpeculativeResourceTuple()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig7isLegalE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">isLegal()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig31kDefaultLookaheadDecodingWindowE"><code class="docutils literal notranslate"><span class="pre">kDefaultLookaheadDecodingWindow</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig30kDefaultLookaheadDecodingNgramE"><code class="docutils literal notranslate"><span class="pre">kDefaultLookaheadDecodingNgram</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig40kDefaultLookaheadDecodingVerificationSetE"><code class="docutils literal notranslate"><span class="pre">kDefaultLookaheadDecodingVerificationSet</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig11mWindowSizeE"><code class="docutils literal notranslate"><span class="pre">mWindowSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig10mNgramSizeE"><code class="docutils literal notranslate"><span class="pre">mNgramSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfig20mVerificationSetSizeE"><code class="docutils literal notranslate"><span class="pre">mVerificationSetSize</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10LoraConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::LoraConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig10LoraConfigE6IdTypeNSt8optionalI6TensorEENSt8optionalI6TensorEE"><code class="docutils literal notranslate"><span class="pre">LoraConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig9getTaskIdEv"><code class="docutils literal notranslate"><span class="pre">getTaskId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig10getWeightsEv"><code class="docutils literal notranslate"><span class="pre">getWeights()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor10LoraConfig9getConfigEv"><code class="docutils literal notranslate"><span class="pre">getConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig7mTaskIdE"><code class="docutils literal notranslate"><span class="pre">mTaskId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig8mWeightsE"><code class="docutils literal notranslate"><span class="pre">mWeights</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor10LoraConfig7mConfigE"><code class="docutils literal notranslate"><span class="pre">mConfig</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11MropeConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::MropeConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig11MropeConfigE6Tensor10SizeType32"><code class="docutils literal notranslate"><span class="pre">MropeConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11MropeConfig20getMRopeRotaryCosSinEv"><code class="docutils literal notranslate"><span class="pre">getMRopeRotaryCosSin()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor11MropeConfig22getMRopePositionDeltasEv"><code class="docutils literal notranslate"><span class="pre">getMRopePositionDeltas()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig18mMRopeRotaryCosSinE"><code class="docutils literal notranslate"><span class="pre">mMRopeRotaryCosSin</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor11MropeConfig20mMRopePositionDeltasE"><code class="docutils literal notranslate"><span class="pre">mMRopePositionDeltas</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::MultimodalInput</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput15MultimodalInputENSt6vectorINSt6vectorI10SizeType32EEEENSt6vectorI10SizeType32EENSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">MultimodalInput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput19getMultimodalHashesEv"><code class="docutils literal notranslate"><span class="pre">getMultimodalHashes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput22getMultimodalPositionsEv"><code class="docutils literal notranslate"><span class="pre">getMultimodalPositions()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15MultimodalInput20getMultimodalLengthsEv"><code class="docutils literal notranslate"><span class="pre">getMultimodalLengths()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput17mMultimodalHashesE"><code class="docutils literal notranslate"><span class="pre">mMultimodalHashes</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput20mMultimodalPositionsE"><code class="docutils literal notranslate"><span class="pre">mMultimodalPositions</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15MultimodalInput18mMultimodalLengthsE"><code class="docutils literal notranslate"><span class="pre">mMultimodalLengths</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::OrchestratorConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig18OrchestratorConfigEbNSt6stringENSt10shared_ptrIN3mpi7MpiCommEEEb"><code class="docutils literal notranslate"><span class="pre">OrchestratorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getIsOrchestratorEv"><code class="docutils literal notranslate"><span class="pre">getIsOrchestrator()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig23getWorkerExecutablePathEv"><code class="docutils literal notranslate"><span class="pre">getWorkerExecutablePath()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getOrchLeaderCommEv"><code class="docutils literal notranslate"><span class="pre">getOrchLeaderComm()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18OrchestratorConfig17getSpawnProcessesEv"><code class="docutils literal notranslate"><span class="pre">getSpawnProcesses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setIsOrchestratorEb"><code class="docutils literal notranslate"><span class="pre">setIsOrchestrator()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig23setWorkerExecutablePathERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">setWorkerExecutablePath()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setOrchLeaderCommERKNSt10shared_ptrIN3mpi7MpiCommEEE"><code class="docutils literal notranslate"><span class="pre">setOrchLeaderComm()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig17setSpawnProcessesEb"><code class="docutils literal notranslate"><span class="pre">setSpawnProcesses()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mIsOrchestratorE"><code class="docutils literal notranslate"><span class="pre">mIsOrchestrator</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig21mWorkerExecutablePathE"><code class="docutils literal notranslate"><span class="pre">mWorkerExecutablePath</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mOrchLeaderCommE"><code class="docutils literal notranslate"><span class="pre">mOrchLeaderComm</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18OrchestratorConfig15mSpawnProcessesE"><code class="docutils literal notranslate"><span class="pre">mSpawnProcesses</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::OutputConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig12OutputConfigEbbbbbbNSt8optionalINSt6vectorI21AdditionalModelOutputEEEE"><code class="docutils literal notranslate"><span class="pre">OutputConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig14returnLogProbsE"><code class="docutils literal notranslate"><span class="pre">returnLogProbs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig19returnContextLogitsE"><code class="docutils literal notranslate"><span class="pre">returnContextLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22returnGenerationLogitsE"><code class="docutils literal notranslate"><span class="pre">returnGenerationLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22excludeInputFromOutputE"><code class="docutils literal notranslate"><span class="pre">excludeInputFromOutput</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig19returnEncoderOutputE"><code class="docutils literal notranslate"><span class="pre">returnEncoderOutput</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig17returnPerfMetricsE"><code class="docutils literal notranslate"><span class="pre">returnPerfMetrics</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor12OutputConfig22additionalModelOutputsE"><code class="docutils literal notranslate"><span class="pre">additionalModelOutputs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::ParallelConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig14ParallelConfigE17CommunicationType17CommunicationModeNSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt6vectorI10SizeType32EEEERKNSt8optionalI18OrchestratorConfigEENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">ParallelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationTypeEv"><code class="docutils literal notranslate"><span class="pre">getCommunicationType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig20getCommunicationModeEv"><code class="docutils literal notranslate"><span class="pre">getCommunicationMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig12getDeviceIdsEv"><code class="docutils literal notranslate"><span class="pre">getDeviceIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig17getParticipantIdsEv"><code class="docutils literal notranslate"><span class="pre">getParticipantIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig21getOrchestratorConfigEv"><code class="docutils literal notranslate"><span class="pre">getOrchestratorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14ParallelConfig11getNumNodesEv"><code class="docutils literal notranslate"><span class="pre">getNumNodes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationTypeE17CommunicationType"><code class="docutils literal notranslate"><span class="pre">setCommunicationType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig20setCommunicationModeE17CommunicationMode"><code class="docutils literal notranslate"><span class="pre">setCommunicationMode()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig12setDeviceIdsERKNSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setDeviceIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig17setParticipantIdsERKNSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setParticipantIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig21setOrchestratorConfigERK18OrchestratorConfig"><code class="docutils literal notranslate"><span class="pre">setOrchestratorConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig11setNumNodesE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setNumNodes()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommTypeE"><code class="docutils literal notranslate"><span class="pre">mCommType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mCommModeE"><code class="docutils literal notranslate"><span class="pre">mCommMode</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig10mDeviceIdsE"><code class="docutils literal notranslate"><span class="pre">mDeviceIds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig15mParticipantIdsE"><code class="docutils literal notranslate"><span class="pre">mParticipantIds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig19mOrchestratorConfigE"><code class="docutils literal notranslate"><span class="pre">mOrchestratorConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14ParallelConfig9mNumNodesE"><code class="docutils literal notranslate"><span class="pre">mNumNodes</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::PeftCacheConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15PeftCacheConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalIfEERKNSt8optionalI6size_tEERKNSt8optionalINSt6stringEEE"><code class="docutils literal notranslate"><span class="pre">PeftCacheConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfigeqERK15PeftCacheConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getNumHostModuleLayerEv"><code class="docutils literal notranslate"><span class="pre">getNumHostModuleLayer()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getNumDeviceModuleLayerEv"><code class="docutils literal notranslate"><span class="pre">getNumDeviceModuleLayer()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getOptimalAdapterSizeEv"><code class="docutils literal notranslate"><span class="pre">getOptimalAdapterSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getMaxAdapterSizeEv"><code class="docutils literal notranslate"><span class="pre">getMaxAdapterSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getNumPutWorkersEv"><code class="docutils literal notranslate"><span class="pre">getNumPutWorkers()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig19getNumEnsureWorkersEv"><code class="docutils literal notranslate"><span class="pre">getNumEnsureWorkers()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig17getNumCopyStreamsEv"><code class="docutils literal notranslate"><span class="pre">getNumCopyStreams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig23getMaxPagesPerBlockHostEv"><code class="docutils literal notranslate"><span class="pre">getMaxPagesPerBlockHost()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig25getMaxPagesPerBlockDeviceEv"><code class="docutils literal notranslate"><span class="pre">getMaxPagesPerBlockDevice()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig21getDeviceCachePercentEv"><code class="docutils literal notranslate"><span class="pre">getDeviceCachePercent()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig16getHostCacheSizeEv"><code class="docutils literal notranslate"><span class="pre">getHostCacheSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15PeftCacheConfig18getLoraPrefetchDirEv"><code class="docutils literal notranslate"><span class="pre">getLoraPrefetchDir()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig26kDefaultOptimalAdapterSizeE"><code class="docutils literal notranslate"><span class="pre">kDefaultOptimalAdapterSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig22kDefaultMaxAdapterSizeE"><code class="docutils literal notranslate"><span class="pre">kDefaultMaxAdapterSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig28kDefaultMaxPagesPerBlockHostE"><code class="docutils literal notranslate"><span class="pre">kDefaultMaxPagesPerBlockHost</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig30kDefaultMaxPagesPerBlockDeviceE"><code class="docutils literal notranslate"><span class="pre">kDefaultMaxPagesPerBlockDevice</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mNumHostModuleLayerE"><code class="docutils literal notranslate"><span class="pre">mNumHostModuleLayer</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mNumDeviceModuleLayerE"><code class="docutils literal notranslate"><span class="pre">mNumDeviceModuleLayer</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mOptimalAdapterSizeE"><code class="docutils literal notranslate"><span class="pre">mOptimalAdapterSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mMaxAdapterSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxAdapterSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mNumPutWorkersE"><code class="docutils literal notranslate"><span class="pre">mNumPutWorkers</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig17mNumEnsureWorkersE"><code class="docutils literal notranslate"><span class="pre">mNumEnsureWorkers</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig15mNumCopyStreamsE"><code class="docutils literal notranslate"><span class="pre">mNumCopyStreams</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig21mMaxPagesPerBlockHostE"><code class="docutils literal notranslate"><span class="pre">mMaxPagesPerBlockHost</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig23mMaxPagesPerBlockDeviceE"><code class="docutils literal notranslate"><span class="pre">mMaxPagesPerBlockDevice</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig19mDeviceCachePercentE"><code class="docutils literal notranslate"><span class="pre">mDeviceCachePercent</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig14mHostCacheSizeE"><code class="docutils literal notranslate"><span class="pre">mHostCacheSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15PeftCacheConfig16mLoraPrefetchDirE"><code class="docutils literal notranslate"><span class="pre">mLoraPrefetchDir</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::PromptTuningConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig18PromptTuningConfigE6TensorNSt8optionalI16VecTokenExtraIdsEE"><code class="docutils literal notranslate"><span class="pre">PromptTuningConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig17getEmbeddingTableEv"><code class="docutils literal notranslate"><span class="pre">getEmbeddingTable()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor18PromptTuningConfig21getInputTokenExtraIdsEv"><code class="docutils literal notranslate"><span class="pre">getInputTokenExtraIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig15mEmbeddingTableE"><code class="docutils literal notranslate"><span class="pre">mEmbeddingTable</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor18PromptTuningConfig19mInputTokenExtraIdsE"><code class="docutils literal notranslate"><span class="pre">mInputTokenExtraIds</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7RequestE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Request</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestE9VecTokens10SizeType32bRK14SamplingConfigRK12OutputConfigRKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalINSt4listI9VecTokensEEEENSt8optionalI6TensorEENSt8optionalI25ExternalDraftTokensConfigEENSt8optionalI18PromptTuningConfigEENSt8optionalI15MultimodalInputEENSt8optionalI6TensorEENSt8optionalI11MropeConfigEENSt8optionalI10LoraConfigEENSt8optionalI23LookaheadDecodingConfigEENSt8optionalI22KvCacheRetentionConfigEENSt8optionalINSt6stringEEENSt8optionalI19LogitsPostProcessorEENSt8optionalI9VecTokensEENSt8optionalI6IdTypeEEb12PriorityType11RequestTypeNSt8optionalI18ContextPhaseParamsEENSt8optionalI6TensorEENSt8optionalI10SizeType32EENSt8optionalI6TensorEE10SizeType32NSt8optionalI11EagleConfigEENSt8optionalI6TensorEENSt8optionalI20GuidedDecodingParamsEENSt8optionalI10SizeType32EENSt8optionalI16MillisecondsTypeEENSt8optionalI15CacheSaltIDTypeEE"><code class="docutils literal notranslate"><span class="pre">Request()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERK7Request"><code class="docutils literal notranslate"><span class="pre">Request()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request7RequestERR7Request"><code class="docutils literal notranslate"><span class="pre">Request()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7RequestaSERK7Request"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7RequestaSERR7Request"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7RequestD0Ev"><code class="docutils literal notranslate"><span class="pre">~Request()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request16getInputTokenIdsEv"><code class="docutils literal notranslate"><span class="pre">getInputTokenIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request12getMaxTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request12getStreamingEv"><code class="docutils literal notranslate"><span class="pre">getStreaming()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request17getSamplingConfigEv"><code class="docutils literal notranslate"><span class="pre">getSamplingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request15getOutputConfigEv"><code class="docutils literal notranslate"><span class="pre">getOutputConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request8getEndIdEv"><code class="docutils literal notranslate"><span class="pre">getEndId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request8getPadIdEv"><code class="docutils literal notranslate"><span class="pre">getPadId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request14getPositionIdsEv"><code class="docutils literal notranslate"><span class="pre">getPositionIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request11getBadWordsEv"><code class="docutils literal notranslate"><span class="pre">getBadWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request12getStopWordsEv"><code class="docutils literal notranslate"><span class="pre">getStopWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request16getEmbeddingBiasEv"><code class="docutils literal notranslate"><span class="pre">getEmbeddingBias()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request28getExternalDraftTokensConfigEv"><code class="docutils literal notranslate"><span class="pre">getExternalDraftTokensConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request21getPromptTuningConfigEv"><code class="docutils literal notranslate"><span class="pre">getPromptTuningConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request18getMultimodalInputEv"><code class="docutils literal notranslate"><span class="pre">getMultimodalInput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request22getMultimodalEmbeddingEv"><code class="docutils literal notranslate"><span class="pre">getMultimodalEmbedding()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request14getMropeConfigEv"><code class="docutils literal notranslate"><span class="pre">getMropeConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request13getLoraConfigEv"><code class="docutils literal notranslate"><span class="pre">getLoraConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request18getLookaheadConfigEv"><code class="docutils literal notranslate"><span class="pre">getLookaheadConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request25getKvCacheRetentionConfigEv"><code class="docutils literal notranslate"><span class="pre">getKvCacheRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request26getLogitsPostProcessorNameEv"><code class="docutils literal notranslate"><span class="pre">getLogitsPostProcessorName()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request22getLogitsPostProcessorEv"><code class="docutils literal notranslate"><span class="pre">getLogitsPostProcessor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputTokenIdsEv"><code class="docutils literal notranslate"><span class="pre">getEncoderInputTokenIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request11getClientIdEv"><code class="docutils literal notranslate"><span class="pre">getClientId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request11getPriorityEv"><code class="docutils literal notranslate"><span class="pre">getPriority()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request27getReturnAllGeneratedTokensEv"><code class="docutils literal notranslate"><span class="pre">getReturnAllGeneratedTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request21getContextPhaseParamsEv"><code class="docutils literal notranslate"><span class="pre">getContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request23getEncoderInputFeaturesEv"><code class="docutils literal notranslate"><span class="pre">getEncoderInputFeatures()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request22getEncoderOutputLengthEv"><code class="docutils literal notranslate"><span class="pre">getEncoderOutputLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request21getCrossAttentionMaskEv"><code class="docutils literal notranslate"><span class="pre">getCrossAttentionMask()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request14getRequestTypeEv"><code class="docutils literal notranslate"><span class="pre">getRequestType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request14getEagleConfigEv"><code class="docutils literal notranslate"><span class="pre">getEagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request22getSkipCrossAttnBlocksEv"><code class="docutils literal notranslate"><span class="pre">getSkipCrossAttnBlocks()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request23getGuidedDecodingParamsEv"><code class="docutils literal notranslate"><span class="pre">getGuidedDecodingParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request21getLanguageAdapterUidEv"><code class="docutils literal notranslate"><span class="pre">getLanguageAdapterUid()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request17getAllottedTimeMsEv"><code class="docutils literal notranslate"><span class="pre">getAllottedTimeMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request14getCacheSaltIDEv"><code class="docutils literal notranslate"><span class="pre">getCacheSaltID()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor7Request24getAdditionalOutputNamesEv"><code class="docutils literal notranslate"><span class="pre">getAdditionalOutputNames()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request12setStreamingEb"><code class="docutils literal notranslate"><span class="pre">setStreaming()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request17setSamplingConfigERK14SamplingConfig"><code class="docutils literal notranslate"><span class="pre">setSamplingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request15setOutputConfigERK12OutputConfig"><code class="docutils literal notranslate"><span class="pre">setOutputConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request8setEndIdE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setEndId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request8setPadIdE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setPadId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request14setPositionIdsERKNSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setPositionIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request11setBadWordsERKNSt4listI9VecTokensEE"><code class="docutils literal notranslate"><span class="pre">setBadWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request12setStopWordsERKNSt4listI9VecTokensEE"><code class="docutils literal notranslate"><span class="pre">setStopWords()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request16setEmbeddingBiasERK6Tensor"><code class="docutils literal notranslate"><span class="pre">setEmbeddingBias()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request28setExternalDraftTokensConfigERK25ExternalDraftTokensConfig"><code class="docutils literal notranslate"><span class="pre">setExternalDraftTokensConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request21setPromptTuningConfigERK18PromptTuningConfig"><code class="docutils literal notranslate"><span class="pre">setPromptTuningConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request22setMultimodalEmbeddingERK6Tensor"><code class="docutils literal notranslate"><span class="pre">setMultimodalEmbedding()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request18setMultimodalInputERK15MultimodalInput"><code class="docutils literal notranslate"><span class="pre">setMultimodalInput()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request14setMropeConfigERK11MropeConfig"><code class="docutils literal notranslate"><span class="pre">setMropeConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request13setLoraConfigERK10LoraConfig"><code class="docutils literal notranslate"><span class="pre">setLoraConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request18setLookaheadConfigERK23LookaheadDecodingConfig"><code class="docutils literal notranslate"><span class="pre">setLookaheadConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request25setKvCacheRetentionConfigERK22KvCacheRetentionConfig"><code class="docutils literal notranslate"><span class="pre">setKvCacheRetentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request26setLogitsPostProcessorNameERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">setLogitsPostProcessorName()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request22setLogitsPostProcessorERKNSt8optionalI19LogitsPostProcessorEE"><code class="docutils literal notranslate"><span class="pre">setLogitsPostProcessor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputTokenIdsERK9VecTokens"><code class="docutils literal notranslate"><span class="pre">setEncoderInputTokenIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request11setClientIdE6IdType"><code class="docutils literal notranslate"><span class="pre">setClientId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request11setPriorityE12PriorityType"><code class="docutils literal notranslate"><span class="pre">setPriority()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request27setReturnAllGeneratedTokensEb"><code class="docutils literal notranslate"><span class="pre">setReturnAllGeneratedTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request14setRequestTypeERK11RequestType"><code class="docutils literal notranslate"><span class="pre">setRequestType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request21setContextPhaseParamsE18ContextPhaseParams"><code class="docutils literal notranslate"><span class="pre">setContextPhaseParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request23setEncoderInputFeaturesE6Tensor"><code class="docutils literal notranslate"><span class="pre">setEncoderInputFeatures()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request22setEncoderOutputLengthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setEncoderOutputLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request21setCrossAttentionMaskE6Tensor"><code class="docutils literal notranslate"><span class="pre">setCrossAttentionMask()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request14setEagleConfigERKNSt8optionalI11EagleConfigEE"><code class="docutils literal notranslate"><span class="pre">setEagleConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request22setSkipCrossAttnBlocksE6Tensor"><code class="docutils literal notranslate"><span class="pre">setSkipCrossAttnBlocks()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request23setGuidedDecodingParamsERK20GuidedDecodingParams"><code class="docutils literal notranslate"><span class="pre">setGuidedDecodingParams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request21setLanguageAdapterUidE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setLanguageAdapterUid()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request17setAllottedTimeMsE16MillisecondsType"><code class="docutils literal notranslate"><span class="pre">setAllottedTimeMs()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request14setCacheSaltIDE15CacheSaltIDType"><code class="docutils literal notranslate"><span class="pre">setCacheSaltID()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request16kDefaultPriorityE"><code class="docutils literal notranslate"><span class="pre">kDefaultPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request25kBatchedPostProcessorNameE"><code class="docutils literal notranslate"><span class="pre">kBatchedPostProcessorName</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request31kDynamicPostProcessorNamePrefixE"><code class="docutils literal notranslate"><span class="pre">kDynamicPostProcessorNamePrefix</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor7Request5mImplE"><code class="docutils literal notranslate"><span class="pre">mImpl</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ResponseE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Response</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdTypeNSt6stringENSt8optionalI6IdTypeEE"><code class="docutils literal notranslate"><span class="pre">Response()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseE6IdType6ResultNSt8optionalI6IdTypeEE"><code class="docutils literal notranslate"><span class="pre">Response()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ResponseD0Ev"><code class="docutils literal notranslate"><span class="pre">~Response()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERK8Response"><code class="docutils literal notranslate"><span class="pre">Response()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Response8ResponseERR8Response"><code class="docutils literal notranslate"><span class="pre">Response()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ResponseaSERK8Response"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8ResponseaSERR8Response"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Response12getRequestIdEv"><code class="docutils literal notranslate"><span class="pre">getRequestId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Response11getClientIdEv"><code class="docutils literal notranslate"><span class="pre">getClientId()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Response8hasErrorEv"><code class="docutils literal notranslate"><span class="pre">hasError()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Response11getErrorMsgEv"><code class="docutils literal notranslate"><span class="pre">getErrorMsg()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8Response9getResultEv"><code class="docutils literal notranslate"><span class="pre">getResult()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8Response5mImplE"><code class="docutils literal notranslate"><span class="pre">mImpl</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6ResultE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::Result</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result7isFinalE"><code class="docutils literal notranslate"><span class="pre">isFinal</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result14outputTokenIdsE"><code class="docutils literal notranslate"><span class="pre">outputTokenIds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result11cumLogProbsE"><code class="docutils literal notranslate"><span class="pre">cumLogProbs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result8logProbsE"><code class="docutils literal notranslate"><span class="pre">logProbs</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result13contextLogitsE"><code class="docutils literal notranslate"><span class="pre">contextLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result16generationLogitsE"><code class="docutils literal notranslate"><span class="pre">generationLogits</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result21specDecFastLogitsInfoE"><code class="docutils literal notranslate"><span class="pre">specDecFastLogitsInfo</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result13encoderOutputE"><code class="docutils literal notranslate"><span class="pre">encoderOutput</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result13finishReasonsE"><code class="docutils literal notranslate"><span class="pre">finishReasons</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result18contextPhaseParamsE"><code class="docutils literal notranslate"><span class="pre">contextPhaseParams</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result12decodingIterE"><code class="docutils literal notranslate"><span class="pre">decodingIter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result23avgDecodedTokensPerIterE"><code class="docutils literal notranslate"><span class="pre">avgDecodedTokensPerIter</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result13sequenceIndexE"><code class="docutils literal notranslate"><span class="pre">sequenceIndex</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result15isSequenceFinalE"><code class="docutils literal notranslate"><span class="pre">isSequenceFinal</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result18requestPerfMetricsE"><code class="docutils literal notranslate"><span class="pre">requestPerfMetrics</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor6Result17additionalOutputsE"><code class="docutils literal notranslate"><span class="pre">additionalOutputs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDurationE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::RetentionPriorityAndDuration</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration28RetentionPriorityAndDurationERKNSt8optionalI17RetentionPriorityEERKNSt8optionalINSt6chrono12millisecondsEEE"><code class="docutils literal notranslate"><span class="pre">RetentionPriorityAndDuration()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration17retentionPriorityE"><code class="docutils literal notranslate"><span class="pre">retentionPriority</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor28RetentionPriorityAndDuration10durationMsE"><code class="docutils literal notranslate"><span class="pre">durationMs</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::SamplingConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14SamplingConfigE10SizeType32RKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI11TokenIdTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI14RandomSeedTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI10SizeType32EERKNSt8optionalI9FloatTypeEERKNSt8optionalINSt6vectorI10SizeType32EEEE"><code class="docutils literal notranslate"><span class="pre">SamplingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfigeqERK14SamplingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getBeamWidthEv"><code class="docutils literal notranslate"><span class="pre">getBeamWidth()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getNumReturnBeamsEv"><code class="docutils literal notranslate"><span class="pre">getNumReturnBeams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopKEv"><code class="docutils literal notranslate"><span class="pre">getTopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getTopPEv"><code class="docutils literal notranslate"><span class="pre">getTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig10getTopPMinEv"><code class="docutils literal notranslate"><span class="pre">getTopPMin()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig15getTopPResetIdsEv"><code class="docutils literal notranslate"><span class="pre">getTopPResetIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getTopPDecayEv"><code class="docutils literal notranslate"><span class="pre">getTopPDecay()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getSeedEv"><code class="docutils literal notranslate"><span class="pre">getSeed()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig14getTemperatureEv"><code class="docutils literal notranslate"><span class="pre">getTemperature()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig12getMinTokensEv"><code class="docutils literal notranslate"><span class="pre">getMinTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig26getBeamSearchDiversityRateEv"><code class="docutils literal notranslate"><span class="pre">getBeamSearchDiversityRate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getRepetitionPenaltyEv"><code class="docutils literal notranslate"><span class="pre">getRepetitionPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig18getPresencePenaltyEv"><code class="docutils literal notranslate"><span class="pre">getPresencePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig19getFrequencyPenaltyEv"><code class="docutils literal notranslate"><span class="pre">getFrequencyPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getPromptIgnoreLengthEv"><code class="docutils literal notranslate"><span class="pre">getPromptIgnoreLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getLengthPenaltyEv"><code class="docutils literal notranslate"><span class="pre">getLengthPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig16getEarlyStoppingEv"><code class="docutils literal notranslate"><span class="pre">getEarlyStopping()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig20getNoRepeatNgramSizeEv"><code class="docutils literal notranslate"><span class="pre">getNoRepeatNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig21getNumReturnSequencesEv"><code class="docutils literal notranslate"><span class="pre">getNumReturnSequences()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig7getMinPEv"><code class="docutils literal notranslate"><span class="pre">getMinP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor14SamplingConfig17getBeamWidthArrayEv"><code class="docutils literal notranslate"><span class="pre">getBeamWidthArray()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setBeamWidthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setBeamWidth()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopKERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setTopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setTopPERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10setTopPMinERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setTopPMin()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15setTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"><code class="docutils literal notranslate"><span class="pre">setTopPResetIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setTopPDecayERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setTopPDecay()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setSeedERKNSt8optionalI14RandomSeedTypeEE"><code class="docutils literal notranslate"><span class="pre">setSeed()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14setTemperatureERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setTemperature()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12setMinTokensERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setMinTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig26setBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setBeamSearchDiversityRate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20setRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setRepetitionPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18setPresencePenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setPresencePenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19setFrequencyPenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setFrequencyPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig21setPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setPromptIgnoreLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16setLengthPenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setLengthPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16setEarlyStoppingERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setEarlyStopping()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20setNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setNoRepeatNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig21setNumReturnSequencesERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setNumReturnSequences()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig7setMinPERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">setMinP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17setBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEE"><code class="docutils literal notranslate"><span class="pre">setBeamWidthArray()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig20updateNumReturnBeamsEv"><code class="docutils literal notranslate"><span class="pre">updateNumReturnBeams()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mBeamWidthE"><code class="docutils literal notranslate"><span class="pre">mBeamWidth</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopKE"><code class="docutils literal notranslate"><span class="pre">mTopK</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mTopPE"><code class="docutils literal notranslate"><span class="pre">mTopP</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig8mTopPMinE"><code class="docutils literal notranslate"><span class="pre">mTopPMin</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig13mTopPResetIdsE"><code class="docutils literal notranslate"><span class="pre">mTopPResetIds</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mTopPDecayE"><code class="docutils literal notranslate"><span class="pre">mTopPDecay</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mSeedE"><code class="docutils literal notranslate"><span class="pre">mSeed</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12mTemperatureE"><code class="docutils literal notranslate"><span class="pre">mTemperature</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig10mMinTokensE"><code class="docutils literal notranslate"><span class="pre">mMinTokens</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig24mBeamSearchDiversityRateE"><code class="docutils literal notranslate"><span class="pre">mBeamSearchDiversityRate</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18mRepetitionPenaltyE"><code class="docutils literal notranslate"><span class="pre">mRepetitionPenalty</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16mPresencePenaltyE"><code class="docutils literal notranslate"><span class="pre">mPresencePenalty</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17mFrequencyPenaltyE"><code class="docutils literal notranslate"><span class="pre">mFrequencyPenalty</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19mPromptIgnoreLengthE"><code class="docutils literal notranslate"><span class="pre">mPromptIgnoreLength</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14mLengthPenaltyE"><code class="docutils literal notranslate"><span class="pre">mLengthPenalty</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14mEarlyStoppingE"><code class="docutils literal notranslate"><span class="pre">mEarlyStopping</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18mNoRepeatNgramSizeE"><code class="docutils literal notranslate"><span class="pre">mNoRepeatNgramSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19mNumReturnSequencesE"><code class="docutils literal notranslate"><span class="pre">mNumReturnSequences</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15mNumReturnBeamsE"><code class="docutils literal notranslate"><span class="pre">mNumReturnBeams</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig5mMinPE"><code class="docutils literal notranslate"><span class="pre">mMinP</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig15mBeamWidthArrayE"><code class="docutils literal notranslate"><span class="pre">mBeamWidthArray</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkBeamWidthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">checkBeamWidth()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopKERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTopK()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkTopPERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTopP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig12checkTopPMinERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTopPMin()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig17checkTopPResetIdsERKNSt8optionalI11TokenIdTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTopPResetIds()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkTopPDecayERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTopPDecay()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig16checkTemperatureERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkTemperature()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig14checkMinTokensERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">checkMinTokens()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig28checkBeamSearchDiversityRateERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkBeamSearchDiversityRate()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkRepetitionPenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkRepetitionPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkPromptIgnoreLengthERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">checkPromptIgnoreLength()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkLengthPenaltyERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkLengthPenalty()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig18checkEarlyStoppingERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">checkEarlyStopping()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig22checkNoRepeatNgramSizeERKNSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">checkNoRepeatNgramSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig23checkNumReturnSequencesERKNSt8optionalI10SizeType32EE10SizeType32"><code class="docutils literal notranslate"><span class="pre">checkNumReturnSequences()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig9checkMinPERKNSt8optionalI9FloatTypeEE"><code class="docutils literal notranslate"><span class="pre">checkMinP()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor14SamplingConfig19checkBeamWidthArrayERKNSt8optionalINSt6vectorI10SizeType32EEEEK10SizeType32"><code class="docutils literal notranslate"><span class="pre">checkBeamWidthArray()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::SchedulerConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig15SchedulerConfigE23CapacitySchedulerPolicyNSt8optionalI21ContextChunkingPolicyEENSt8optionalI18DynamicBatchConfigEE"><code class="docutils literal notranslate"><span class="pre">SchedulerConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfigeqERK15SchedulerConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig26getCapacitySchedulerPolicyEv"><code class="docutils literal notranslate"><span class="pre">getCapacitySchedulerPolicy()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig24getContextChunkingPolicyEv"><code class="docutils literal notranslate"><span class="pre">getContextChunkingPolicy()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor15SchedulerConfig21getDynamicBatchConfigEv"><code class="docutils literal notranslate"><span class="pre">getDynamicBatchConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig24mCapacitySchedulerPolicyE"><code class="docutils literal notranslate"><span class="pre">mCapacitySchedulerPolicy</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig22mContextChunkingPolicyE"><code class="docutils literal notranslate"><span class="pre">mContextChunkingPolicy</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor15SchedulerConfig19mDynamicBatchConfigE"><code class="docutils literal notranslate"><span class="pre">mDynamicBatchConfig</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::SpeculativeDecodingConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig25SpeculativeDecodingConfigEb"><code class="docutils literal notranslate"><span class="pre">SpeculativeDecodingConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor25SpeculativeDecodingConfigeqERK25SpeculativeDecodingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor25SpeculativeDecodingConfig10fastLogitsE"><code class="docutils literal notranslate"><span class="pre">fastLogits</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfoE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::SpeculativeDecodingFastLogitsInfo</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo8toTensorEv"><code class="docutils literal notranslate"><span class="pre">toTensor()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo14draftRequestIdE"><code class="docutils literal notranslate"><span class="pre">draftRequestId</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor33SpeculativeDecodingFastLogitsInfo18draftParticipantIdE"><code class="docutils literal notranslate"><span class="pre">draftParticipantId</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm3mpiE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::mpi</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#datatransceiverstate-h">dataTransceiverState.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::DataTransceiverState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEv"><code class="docutils literal notranslate"><span class="pre">DataTransceiverState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState20DataTransceiverStateEN8kv_cache10CacheStateEN8kv_cache9CommStateE"><code class="docutils literal notranslate"><span class="pre">DataTransceiverState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState13setCacheStateEN8kv_cache10CacheStateE"><code class="docutils literal notranslate"><span class="pre">setCacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState13getCacheStateEv"><code class="docutils literal notranslate"><span class="pre">getCacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState12setCommStateEN8kv_cache9CommStateE"><code class="docutils literal notranslate"><span class="pre">setCommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState12getCommStateEv"><code class="docutils literal notranslate"><span class="pre">getCommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverStateeqERK20DataTransceiverState"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor20DataTransceiverState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState11mCacheStateE"><code class="docutils literal notranslate"><span class="pre">mCacheState</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor20DataTransceiverState10mCommStateE"><code class="docutils literal notranslate"><span class="pre">mCommState</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::AgentState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateENSt6stringENSt6stringE"><code class="docutils literal notranslate"><span class="pre">AgentState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10AgentStateEv"><code class="docutils literal notranslate"><span class="pre">AgentState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentStateeqERK10AgentState"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10AgentState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState10mAgentNameE"><code class="docutils literal notranslate"><span class="pre">mAgentName</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10AgentState15mConnectionInfoE"><code class="docutils literal notranslate"><span class="pre">mConnectionInfo</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::CacheState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionTypeE"><code class="docutils literal notranslate"><span class="pre">AttentionType</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType8kDEFAULTE"><code class="docutils literal notranslate"><span class="pre">kDEFAULT</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState13AttentionType4kMLAE"><code class="docutils literal notranslate"><span class="pre">kMLA</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE11ModelConfigRKN7runtime11WorldConfigERKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibb10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">CacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateENSt6vectorI10SizeType32EE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">CacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState10CacheStateE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt6vectorI10SizeType32EEN8nvinfer18DataTypeE13AttentionTypeibiibb10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">CacheState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheStateeqERKN8kv_cache10CacheStateE"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14getModelConfigEv"><code class="docutils literal notranslate"><span class="pre">getModelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState17getParallelConfigEv"><code class="docutils literal notranslate"><span class="pre">getParallelConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState18getAttentionConfigEv"><code class="docutils literal notranslate"><span class="pre">getAttentionConfig()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getEnableBlockReuseEv"><code class="docutils literal notranslate"><span class="pre">getEnableBlockReuse()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState19getHasIndexerKCacheEv"><code class="docutils literal notranslate"><span class="pre">getHasIndexerKCache()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState20getIndexerDimPerHeadEv"><code class="docutils literal notranslate"><span class="pre">getIndexerDimPerHead()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState30getIndexerKCacheQuantBlockSizeEv"><code class="docutils literal notranslate"><span class="pre">getIndexerKCacheQuantBlockSize()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState12mModelConfigE"><code class="docutils literal notranslate"><span class="pre">mModelConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15mParallelConfigE"><code class="docutils literal notranslate"><span class="pre">mParallelConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState9mDataTypeE"><code class="docutils literal notranslate"><span class="pre">mDataType</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState16mAttentionConfigE"><code class="docutils literal notranslate"><span class="pre">mAttentionConfig</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mEnableBlockReuseE"><code class="docutils literal notranslate"><span class="pre">mEnableBlockReuse</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState17mHasIndexerKCacheE"><code class="docutils literal notranslate"><span class="pre">mHasIndexerKCache</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState18mIndexerDimPerHeadE"><code class="docutils literal notranslate"><span class="pre">mIndexerDimPerHead</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState28mIndexerKCacheQuantBlockSizeE"><code class="docutils literal notranslate"><span class="pre">mIndexerKCacheQuantBlockSize</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::CacheState::AttentionConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig15AttentionConfigE13AttentionTypei"><code class="docutils literal notranslate"><span class="pre">AttentionConfig()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfigeqERK15AttentionConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig14mAttentionTypeE"><code class="docutils literal notranslate"><span class="pre">mAttentionType</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState15AttentionConfig9mKvFactorE"><code class="docutils literal notranslate"><span class="pre">mKvFactor</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::CacheState::ModelConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState11ModelConfigeqERK11ModelConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig18mNbKvHeadsPerLayerE"><code class="docutils literal notranslate"><span class="pre">mNbKvHeadsPerLayer</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig12mSizePerHeadE"><code class="docutils literal notranslate"><span class="pre">mSizePerHead</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState11ModelConfig15mTokensPerBlockE"><code class="docutils literal notranslate"><span class="pre">mTokensPerBlock</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::CacheState::ParallelConfig</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfigeqERK14ParallelConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mTensorParallelismE"><code class="docutils literal notranslate"><span class="pre">mTensorParallelism</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig20mPipelineParallelismE"><code class="docutils literal notranslate"><span class="pre">mPipelineParallelism</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig19mContextParallelismE"><code class="docutils literal notranslate"><span class="pre">mContextParallelism</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig18mEnableAttentionDPE"><code class="docutils literal notranslate"><span class="pre">mEnableAttentionDP</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPrankE"><code class="docutils literal notranslate"><span class="pre">mDPrank</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig7mDPsizeE"><code class="docutils literal notranslate"><span class="pre">mDPsize</span></code></a></li>
|
||
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10CacheState14ParallelConfig23mAttentionLayerNumPerPPE"><code class="docutils literal notranslate"><span class="pre">mAttentionLayerNumPerPP</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::CommState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateEv"><code class="docutils literal notranslate"><span class="pre">CommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10SizeType32EEi"><code class="docutils literal notranslate"><span class="pre">CommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI11SocketStateEEi"><code class="docutils literal notranslate"><span class="pre">CommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt8uint16_tENSt6stringE"><code class="docutils literal notranslate"><span class="pre">CommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState9CommStateENSt6vectorI10AgentStateEEi"><code class="docutils literal notranslate"><span class="pre">CommState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10isMpiStateEv"><code class="docutils literal notranslate"><span class="pre">isMpiState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13isSocketStateEv"><code class="docutils literal notranslate"><span class="pre">isSocketState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState12isAgentStateEv"><code class="docutils literal notranslate"><span class="pre">isAgentState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState11getMpiStateEv"><code class="docutils literal notranslate"><span class="pre">getMpiState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState14getSocketStateEv"><code class="docutils literal notranslate"><span class="pre">getSocketState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState13getAgentStateEv"><code class="docutils literal notranslate"><span class="pre">getAgentState()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState10getSelfIdxEv"><code class="docutils literal notranslate"><span class="pre">getSelfIdx()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommStateeqERK9CommState"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache9CommState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState6mStateE"><code class="docutils literal notranslate"><span class="pre">mState</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache9CommState8mSelfIdxE"><code class="docutils literal notranslate"><span class="pre">mSelfIdx</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::MpiState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiStateeqERK8MpiState"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache8MpiState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache8MpiState6mRanksE"><code class="docutils literal notranslate"><span class="pre">mRanks</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::SocketState</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketStateeqERK11SocketState"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11SocketState8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState5mPortE"><code class="docutils literal notranslate"><span class="pre">mPort</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11SocketState3mIpE"><code class="docutils literal notranslate"><span class="pre">mIp</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#cachecommunicator-h">cacheCommunicator.h</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::Connection</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache10ConnectionD0Ev"><code class="docutils literal notranslate"><span class="pre">~Connection()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4sendERK11DataContextPKv6size_t"><code class="docutils literal notranslate"><span class="pre">send()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection4recvERK11DataContextPv6size_t"><code class="docutils literal notranslate"><span class="pre">recv()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache10Connection12isThreadSafeEv"><code class="docutils literal notranslate"><span class="pre">isThreadSafe()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::ConnectionManager</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManagerD0Ev"><code class="docutils literal notranslate"><span class="pre">~ConnectionManager()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager11recvConnectERK11DataContextPv6size_t"><code class="docutils literal notranslate"><span class="pre">recvConnect()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache17ConnectionManager14getConnectionsERK9CommState"><code class="docutils literal notranslate"><span class="pre">getConnections()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache17ConnectionManager12getCommStateEv"><code class="docutils literal notranslate"><span class="pre">getCommState()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContextE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::executor::kv_cache::DataContext</span></code></a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext11DataContextEi"><code class="docutils literal notranslate"><span class="pre">DataContext()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm8executor8kv_cache11DataContext6getTagEv"><code class="docutils literal notranslate"><span class="pre">getTag()</span></code></a></li>
|
||
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm8executor8kv_cache11DataContext4mTagE"><code class="docutils literal notranslate"><span class="pre">mTag</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
</footer>
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
|
||
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
|
||
|
||
|
||
<footer class="bd-footer">
|
||
<div class="bd-footer__inner bd-page-width">
|
||
|
||
<div class="footer-items__start">
|
||
|
||
<div class="footer-item">
|
||
<a class="footer-brand logo" href="https://www.nvidia.com">
|
||
<img src="../_static/nvidia-logo-horiz-rgb-1c-blk-for-screen.svg" class="logo__image only-light" alt="NVIDIA"/>
|
||
<img src="../_static/nvidia-logo-horiz-rgb-1c-wht-for-screen.svg" class="logo__image only-dark" alt="NVIDIA"/>
|
||
</a></div>
|
||
|
||
<div class="footer-item">
|
||
|
||
<div class="footer-links">
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-policy/">Privacy Policy</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-center/">Your Privacy Choices</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/terms-of-service/">Terms of Service</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/accessibility/">Accessibility</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/company-policies/">Corporate Policies</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/product-security/">Product Security</a>
|
||
|
|
||
|
||
|
||
|
||
<a class="external" href="https://www.nvidia.com/en-us/contact/">Contact</a>
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
Copyright © 2025, NVidia.
|
||
<br/>
|
||
|
||
</p>
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
<div class="extra_footer">
|
||
|
||
<p>Last updated on November 23, 2025.</p>
|
||
|
||
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/a761585">a761585</a>.</p>
|
||
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</footer>
|
||
</body>
|
||
</html> |