kanshan/TensorRT-LLMs
1
0
Fork 0
mirror of https://github.com/NVIDIA/TensorRT-LLM.git synced 2026-01-14 06:27:45 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity
33d318a4f9
TensorRT-LLMs/0.20.0/_cpp_gen/runtime.html
Kaiyu Xie 47b1751784 Update GitHub pages to v0.20.0
2025-06-24 03:02:01 +00:00

13760 lines
2.6 MiB
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!DOCTYPE html>
<html lang="en" data-content_root="../" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Runtime &#8212; TensorRT-LLM</title>
<script data-cfasync="false">
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
</script>
<!--
this give us a css class that will be invisible only if js is disabled
-->
<noscript>
<style>
.pst-js-only { display: none !important; }
</style>
</noscript>
<!-- Loaded before other Sphinx assets -->
<link href="../_static/styles/theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=8878045cc6db502f8baf" rel="stylesheet" />
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=8f2a1f02" />
<link rel="stylesheet" type="text/css" href="../_static/styles/nvidia-sphinx-theme.css?v=df3ac72c" />
<link rel="stylesheet" type="text/css" href="../_static/copybutton.css?v=76b2166b" />
<link rel="stylesheet" type="text/css" href="../_static/autodoc_pydantic.css" />
<!-- So that users can add custom icons -->
<script src="../_static/scripts/fontawesome.js?digest=8878045cc6db502f8baf"></script>
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf" />
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
<script src="../_static/doctools.js?v=9a2dae69"></script>
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
<script src="../_static/clipboard.min.js?v=a7894cd8"></script>
<script src="../_static/copybutton.js?v=65e89d2a"></script>
<script>DOCUMENTATION_OPTIONS.pagename = '_cpp_gen/runtime';</script>
<script>
DOCUMENTATION_OPTIONS.theme_version = '0.16.1';
DOCUMENTATION_OPTIONS.theme_switcher_json_url = './_static/switcher.json';
DOCUMENTATION_OPTIONS.theme_switcher_version_match = '0.20.0';
DOCUMENTATION_OPTIONS.show_version_warning_banner =
false;
</script>
<link rel="icon" href="../_static/favicon.png"/>
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="trtllm-build" href="../commands/trtllm-build.html" />
<link rel="prev" title="Executor" href="executor.html" />
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<meta name="docsearch:language" content="en"/>
<meta name="docsearch:version" content="0.20.0" />
</head>
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
<div id="pst-scroll-pixel-helper"></div>
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
<dialog id="pst-search-dialog">
<form class="bd-search d-flex align-items-center"
action="../search.html"
method="get">
<i class="fa-solid fa-magnifying-glass"></i>
<input type="search"
class="form-control"
name="q"
placeholder="Search the docs ..."
aria-label="Search the docs ..."
autocomplete="off"
autocorrect="off"
autocapitalize="off"
spellcheck="false"/>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form>
</dialog>
<div class="pst-async-banner-revealer d-none">
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
<button class="pst-navbar-icon sidebar-toggle primary-toggle" aria-label="Site navigation">
<span class="fa-solid fa-bars"></span>
</button>
<div class="col-lg-3 navbar-header-items__start">
<div class="navbar-item">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT-LLM - Home"/>
<img src="../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT-LLM - Home"/>
<p class="title logo__title">TensorRT-LLM</p>
</a></div>
</div>
<div class="col-lg-9 navbar-header-items">
<div class="me-auto navbar-header-items__center">
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-2"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-2"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-2"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-2">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
</div>
<div class="navbar-header-items__end">
<div class="navbar-item navbar-persistent--container">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="navbar-persistent--mobile">
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="fa-solid fa-magnifying-glass"></i>
<span class="search-button__default-text">Search</span>
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
</button>
</div>
<button class="pst-navbar-icon sidebar-toggle secondary-toggle" aria-label="On this page">
<span class="fa-solid fa-outdent"></span>
</button>
</div>
</header>
<div class="bd-container">
<div class="bd-container__inner bd-page-width">
<dialog id="pst-primary-sidebar-modal"></dialog>
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
<a class="navbar-brand logo" href="../index.html">
<img src="../_static/nvidia-logo-horiz-rgb-blk-for-screen.svg" class="logo__image only-light" alt="TensorRT-LLM - Home"/>
<img src="../_static/nvidia-logo-horiz-rgb-wht-for-screen.svg" class="logo__image only-dark pst-js-only" alt="TensorRT-LLM - Home"/>
<p class="title logo__title">TensorRT-LLM</p>
</a>
<div class="sidebar-header-items sidebar-primary__section">
<div class="sidebar-header-items__center">
<div class="navbar-item">
<div class="version-switcher__container dropdown pst-js-only">
<button id="pst-version-switcher-button-3"
type="button"
class="version-switcher__button btn btn-sm dropdown-toggle"
data-bs-toggle="dropdown"
aria-haspopup="listbox"
aria-controls="pst-version-switcher-list-3"
aria-label="Version switcher list"
>
Choose version <!-- this text may get changed later by javascript -->
<span class="caret"></span>
</button>
<div id="pst-version-switcher-list-3"
class="version-switcher__menu dropdown-menu list-group-flush py-0"
role="listbox" aria-labelledby="pst-version-switcher-button-3">
<!-- dropdown will be populated by javascript on page load -->
</div>
</div></div>
</div>
<div class="sidebar-header-items__end">
<div class="navbar-item">
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
</button></div>
</div>
</div>
<div class="sidebar-primary-items__start sidebar-primary__section">
<div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
aria-label="Table of Contents">
<p class="bd-links__title" role="heading" aria-level="1">Table of Contents</p>
<div class="bd-toc-item navbar-nav"><p aria-level="2" class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../overview.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../quick-start-guide.html">Quick Start Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../key-features.html">Key Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="../torch.html">PyTorch Backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../release-notes.html">Release Notes</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Installation</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../installation/linux.html">Installing on Linux</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation/build-from-source-linux.html">Building from Source Code on Linux</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation/grace-hopper.html">Installing on Grace Hopper</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">LLM API</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../llm-api/index.html">API Introduction</a></li>
<li class="toctree-l1"><a class="reference internal" href="../llm-api/reference.html">API Reference</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Examples</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/index.html">LLM Examples Introduction</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async.html">Generate Text Asynchronously</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_kv_events.html">Get KV Cache Events</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_customize.html">Generate text with customization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_lookahead_decoding.html">Generate Text Using Lookahead Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_medusa_decoding.html">Generate Text Using Medusa Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_quantization.html">Generation with Quantization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference.html">Generate text</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async_streaming.html">Generate Text in Streaming</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_distributed.html">Distributed LLM Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_eagle_decoding.html">Generate Text Using Eagle Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_auto_parallel.html">Automatic Parallelism with LLM</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_llm_distributed.html">Llm Mgmn Llm Distributed</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_bench.html">Llm Mgmn Trtllm Bench</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_serve.html">Llm Mgmn Trtllm Serve</a></li>
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal" href="../examples/customization.html">LLM Common Customizations</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/llm_api_examples.html">LLM Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async.html">Generate Text Asynchronously</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_kv_events.html">Get KV Cache Events</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_customize.html">Generate text with customization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_lookahead_decoding.html">Generate Text Using Lookahead Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_medusa_decoding.html">Generate Text Using Medusa Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_guided_decoding.html">Generate text with guided decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_logits_processor.html">Control generated text using logits processor</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_quantization.html">Generation with Quantization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference.html">Generate text</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_multilora.html">Generate text with multiple LoRA adapters</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_async_streaming.html">Generate Text in Streaming</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_inference_distributed.html">Distributed LLM Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_eagle_decoding.html">Generate Text Using Eagle Decoding</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_auto_parallel.html">Automatic Parallelism with LLM</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_llm_distributed.html">Llm Mgmn Llm Distributed</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_bench.html">Llm Mgmn Trtllm Bench</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/llm_mgmn_trtllm_serve.html">Llm Mgmn Trtllm Serve</a></li>
</ul>
</details></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../examples/trtllm_serve_examples.html">Online Serving Examples</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client.html">Curl Chat Client</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_chat_client_for_multimodal.html">Curl Chat Client For Multimodal</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/curl_completion_client.html">Curl Completion Client</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/deepseek_r1_reasoning_parser.html">Deepseek R1 Reasoning Parser</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client.html">Genai Perf Client</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/genai_perf_client_for_multimodal.html">Genai Perf Client For Multimodal</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_chat_client.html">OpenAI Chat Client</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_chat_client_for_multimodal.html">OpenAI Chat Client</a></li>
<li class="toctree-l2"><a class="reference internal" href="../examples/openai_completion_client.html">OpenAI Completion Client</a></li>
</ul>
</details></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Model Definition API</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.layers.html">Layers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.functional.html">Functionals</a></li>
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.models.html">Models</a></li>
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.plugin.html">Plugin</a></li>
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../python-api/tensorrt_llm.runtime.html">Runtime</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">C++ API</span></p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="executor.html">Executor</a></li>
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Runtime</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Command-Line Reference</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../commands/trtllm-build.html">trtllm-build</a></li>
<li class="toctree-l1"><a class="reference internal" href="../commands/trtllm-serve.html">trtllm-serve</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Architecture</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../architecture/overview.html">TensorRT-LLM Architecture</a></li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/core-concepts.html">Model Definition</a></li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/checkpoint.html">TensorRT-LLM Checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/workflow.html">TensorRT-LLM Build Workflow</a></li>
<li class="toctree-l1"><a class="reference internal" href="../architecture/add-model.html">Adding a Model</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Advanced</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../advanced/gpt-attention.html">Multi-Head, Multi-Query, and Group-Query Attention</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/gpt-runtime.html">C++ GPT Runtime</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/executor.html">Executor API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/graph-rewriting.html">Graph Rewriting Module</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/lora.html">Run gpt-2b + LoRA using Executor / cpp runtime</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/expert-parallelism.html">Expert Parallelism in TensorRT-LLM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/kv-cache-management.html">KV Cache Management: Pools, Blocks, and Events</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/kv-cache-reuse.html">KV cache reuse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/speculative-decoding.html">Speculative Sampling</a></li>
<li class="toctree-l1"><a class="reference internal" href="../advanced/disaggregated-service.html">Disaggregated-Service (experimental)</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Performance</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../performance/perf-overview.html">Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../performance/perf-benchmarking.html">Benchmarking</a></li>
<li class="toctree-l1 has-children"><a class="reference internal" href="../performance/performance-tuning-guide/index.html">Performance Tuning Guide</a><details><summary><span class="toctree-toggle" role="presentation"><i class="fa-solid fa-chevron-down"></i></span></summary><ul>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/benchmarking-default-performance.html">Benchmarking Default Performance</a></li>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/useful-build-time-flags.html">Useful Build-Time Flags</a></li>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/tuning-max-batch-size-and-max-num-tokens.html">Tuning Max Batch Size and Max Num Tokens</a></li>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/deciding-model-sharding-strategy.html">Deciding Model Sharding Strategy</a></li>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/fp8-quantization.html">FP8 Quantization</a></li>
<li class="toctree-l2"><a class="reference internal" href="../performance/performance-tuning-guide/useful-runtime-flags.html">Useful Runtime Options</a></li>
</ul>
</details></li>
<li class="toctree-l1"><a class="reference internal" href="../performance/perf-analysis.html">Performance Analysis</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Reference</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../reference/troubleshooting.html">Troubleshooting</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/support-matrix.html">Support Matrix</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/precision.html">Numerical Precision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/memory.html">Memory Usage of TensorRT-LLM</a></li>
</ul>
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Blogs</span></p>
<ul class="nav bd-sidenav">
<li class="toctree-l1"><a class="reference internal" href="../blogs/H100vsA100.html">H100 has 4.6x A100 Performance in TensorRT-LLM, achieving 10,000 tok/s at 100ms to first token</a></li>
<li class="toctree-l1"><a class="reference internal" href="../blogs/H200launch.html">H200 achieves nearly 12,000 tokens/sec on Llama2-13B with TensorRT-LLM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../blogs/Falcon180B-H200.html">Falcon-180B on a single H200 GPU with INT4 AWQ, and 6.7x faster Llama-70B over A100</a></li>
<li class="toctree-l1"><a class="reference internal" href="../blogs/quantization-in-TRT-LLM.html">Speed up inference with SOTA quantization techniques in TRT-LLM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../blogs/XQA-kernel.html">New XQA-kernel provides 2.4x more Llama-70B throughput within the same latency budget</a></li>
<li class="toctree-l1"><a class="reference internal" href="../blogs/tech_blog/blog1_Pushing_Latency_Boundaries_Optimizing_DeepSeek-R1_Performance_on_NVIDIA_B200_GPUs.html">Pushing Latency Boundaries: Optimizing DeepSeek-R1 Performance on NVIDIA B200 GPUs</a></li>
</ul>
</div>
</nav></div>
</div>
<div class="sidebar-primary-items__end sidebar-primary__section">
</div>
</div>
<main id="main-content" class="bd-main" role="main">
<div class="bd-content">
<div class="bd-article-container">
<div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
<div class="header-article-items__start">
<div class="header-article-item">
<nav aria-label="Breadcrumb" class="d-print-none">
<ul class="bd-breadcrumbs">
<li class="breadcrumb-item breadcrumb-home">
<a href="../index.html" class="nav-link" aria-label="Home">
<i class="fa-solid fa-home"></i>
</a>
</li>
<li class="breadcrumb-item active" aria-current="page"><span class="ellipsis">Runtime</span></li>
</ul>
</nav>
</div>
</div>
</div>
</div>
<div id="searchbox"></div>
<article class="bd-article">
<section id="runtime">
<h1>Runtime<a class="headerlink" href="#runtime" title="Link to this heading">#</a></h1>
<section id="lookaheadmodule-h">
<h2>lookaheadModule.h<a class="headerlink" href="#lookaheadmodule-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv412tensorrt_llm">
<span id="_CPPv312tensorrt_llm"></span><span id="_CPPv212tensorrt_llm"></span><span id="tensorrt_llm"></span><span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimeE">
<span id="_CPPv3N12tensorrt_llm7runtimeE"></span><span id="_CPPv2N12tensorrt_llm7runtimeE"></span><span id="tensorrt_llm::runtime"></span><span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15LookaheadModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime15LookaheadModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime15LookaheadModuleE"></span><span id="tensorrt_llm::runtime::LookaheadModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadModule</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModuleE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleE10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LookaheadModule::LookaheadModule__SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1aa68e4d6dc548f3b5e3e218f8004be9b9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftPathLen</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDecodingDraftTokens</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleEv"></span><span id="tensorrt_llm::runtime::LookaheadModule::LookaheadModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1a78baf71cf60b55802cf94b612644bf78"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15LookaheadModule18setExecutionConfigERKN8executor23LookaheadDecodingConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime15LookaheadModule18setExecutionConfigERKN8executor23LookaheadDecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime15LookaheadModule18setExecutionConfigERKN8executor23LookaheadDecodingConfigE"></span><span id="tensorrt_llm::runtime::LookaheadModule::setExecutionConfig__executor::LookaheadDecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1a38679daa2e10e8ab1f482a38a275fcff"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setExecutionConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule18setExecutionConfigERKN8executor23LookaheadDecodingConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv">
<span id="_CPPv3NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"></span><span id="tensorrt_llm::runtime::LookaheadModule::getExecutionConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1ad81b2560fd286eb36d5083279cd13f13"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getExecutionConfig</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15LookaheadModule16mExecutionConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime15LookaheadModule16mExecutionConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime15LookaheadModule16mExecutionConfigE"></span><span id="tensorrt_llm::runtime::LookaheadModule::mExecutionConfig__executor::LookaheadDecodingConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadModule_1a74f1e9093da62577f4859e740f80896e"></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mExecutionConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule16mExecutionConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="decoderstate-h">
<h2>decoderState.h<a class="headerlink" href="#decoderstate-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoderE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoderE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoderE"></span><span id="tensorrt_llm::runtime::decoder"></span><span class="target" id="namespacetensorrt__llm_1_1runtime_1_1decoder"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">decoder</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="Link to this definition">#</a><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BeamSearchBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers17BeamSearchBuffersERK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffers17BeamSearchBuffersERK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffers17BeamSearchBuffersERK13BufferManager"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers::BeamSearchBuffers__BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers_1aedf06160c454bdb6c17822d4caa6db6f"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BeamSearchBuffers</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers17BeamSearchBuffersERK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7reshapeE10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7reshapeE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7reshapeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers::reshape__SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers_1a396a0e7202600a937ab75ee71b7cc015"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7reshapeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers21mOutputBeamHypothesesE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffers21mOutputBeamHypothesesE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffers21mOutputBeamHypothesesE"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers::mOutputBeamHypotheses__DecodingOutput::BeamHypotheses"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers_1ae036af1fd173e62063edd8559ffe8a8b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE" title="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses"><span class="n"><span class="pre">BeamHypotheses</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOutputBeamHypotheses</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers21mOutputBeamHypothesesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers15mCumLogProbsTmpE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffers15mCumLogProbsTmpE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffers15mCumLogProbsTmpE"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers::mCumLogProbsTmp__DecodingOutput::TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers_1af341a80dc43ada95fa0ff84e18cf0bad"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCumLogProbsTmp</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers15mCumLogProbsTmpE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7mNumSMsE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7mNumSMsE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7mNumSMsE"></span><span id="tensorrt_llm::runtime::decoder::BeamSearchBuffers::mNumSMs__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1BeamSearchBuffers_1ab7d6685e743da84572790f3ae67950f8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumSMs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7mNumSMsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderStateE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderStateE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecoderState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a1b2e94c6a5ae979c4010b5ed02fc2524"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aa0731c61a8980a5857842c059a043f77"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LlmRequestPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">LlmRequest</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1afb54bb6f0e7039a842fb095c8501f5a2"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestVector</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::LlmRequestPtr"><span class="n"><span class="pre">LlmRequestPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a9aa4f08e3e7d307c5ccce6764ceda346"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingInputPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a8ad2479bfe8bae612c5bf01afb84a27b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingOutputPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState12DecoderStateEN8nvinfer18DataTypeERK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState12DecoderStateEN8nvinfer18DataTypeERK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState12DecoderStateEN8nvinfer18DataTypeERK13BufferManager"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::DecoderState__nvinfer1::DataType.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a2826e268239ba03ad37566bb3621f547"></span><span class="sig-name descname"><span class="n"><span class="pre">DecoderState</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState12DecoderStateEN8nvinfer18DataTypeERK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState34allocateSpeculativeDecodingBuffersE23SpeculativeDecodingModeN8nvinfer18DataTypeERK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState34allocateSpeculativeDecodingBuffersE23SpeculativeDecodingModeN8nvinfer18DataTypeERK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState34allocateSpeculativeDecodingBuffersE23SpeculativeDecodingModeN8nvinfer18DataTypeERK13BufferManager"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::allocateSpeculativeDecodingBuffers__SpeculativeDecodingMode.nvinfer1::DataType.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a9328c1c763458972d3b3b8b0c0df7dac"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocateSpeculativeDecodingBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">speculativeDecodingMode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState34allocateSpeculativeDecodingBuffersE23SpeculativeDecodingModeN8nvinfer18DataTypeERK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocate buffers for speculative decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState5setupE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState5setupE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState5setupE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setup__SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.ModelConfigCR.WorldConfigCR.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a3a4dc98ec1af0295fd3c70c1188cb298"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setup</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindow</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState5setupE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24setupSpeculativeDecodingERK23SpeculativeDecodingMode10SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState24setupSpeculativeDecodingERK23SpeculativeDecodingMode10SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState24setupSpeculativeDecodingERK23SpeculativeDecodingMode10SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setupSpeculativeDecoding__SpeculativeDecodingModeCR.SizeType32.ModelConfigCR.WorldConfigCR.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ab806113807ea78b3316a0bce6861a03c"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setupSpeculativeDecoding</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">speculativeDecodingMode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokensPerEngineStep</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24setupSpeculativeDecodingERK23SpeculativeDecodingMode10SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup buffers for speculative decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24setupExplicitDraftTokensEN26ExplicitDraftTokensBuffers6InputsE">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState24setupExplicitDraftTokensEN26ExplicitDraftTokensBuffers6InputsE"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState24setupExplicitDraftTokensEN26ExplicitDraftTokensBuffers6InputsE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setupExplicitDraftTokens__ExplicitDraftTokensBuffers::InputsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a0a23ac6c04636c4b581b0d5b86dceecf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setupExplicitDraftTokens</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">explicitDraftTokensBuffers</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24setupExplicitDraftTokensEN26ExplicitDraftTokensBuffers6InputsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup buffers for ExplicitDraftTokens decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14setupLookaheadE24LookaheadDecodingBuffers">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14setupLookaheadE24LookaheadDecodingBuffers"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14setupLookaheadE24LookaheadDecodingBuffers"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setupLookahead__LookaheadDecodingBuffersC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a98617f124fbe9bd6a6b25a9880cc84a2"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setupLookahead</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers"><span class="n"><span class="pre">LookaheadDecodingBuffers</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">lookaheadDecodingBuffers</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14setupLookaheadE24LookaheadDecodingBuffers" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup buffers for Lookahead decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState10setupEagleEN12EagleBuffers6InputsE">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState10setupEagleEN12EagleBuffers6InputsE"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState10setupEagleEN12EagleBuffers6InputsE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setupEagle__EagleBuffers::InputsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a234f3ff6476354e26bc1d3ab150b1e65"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setupEagle</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">eagleBuffers</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState10setupEagleEN12EagleBuffers6InputsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup buffers for Eagle decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16disableLookaheadERK13RequestVector">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState16disableLookaheadERK13RequestVector"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState16disableLookaheadERK13RequestVector"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::disableLookahead__RequestVectorCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a3cdb65ce4c92a02193e39f6d6cd73606"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookahead</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE" title="tensorrt_llm::runtime::decoder::DecoderState::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">genRequests</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16disableLookaheadERK13RequestVector" title="Link to this definition">#</a><br /></dt>
<dd><p>Disable lookahead decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getFinishedSumEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14getFinishedSumEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14getFinishedSumEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getFinishedSumC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a8ca104c24ff7c985ecf9bb0fa58ffe6d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFinishedSum</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getFinishedSumEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize], number of finished sequences per request, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishReasonsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishReasonsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishReasonsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getFinishReasonsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a4089130cc68b8803b0b706f98a561053"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFinishReasons</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishReasonsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, beamWidth], FinishedState value, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getIdsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a4245396ce8bfc3e3954cab6b24d84243"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth, maxInputLength + maxNewTokens], contains input token ids and generated token ids without padding, on gpu. In case of beam search, contains the ungathered data. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getIds__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a90d5ad9a815fe194ca1d2b1818de639e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>batchIdx</strong> – index of the batch </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>[maxBeamWidth, maxInputLength + maxNewTokens], contains input token ids and generated token ids without padding for request <code class="docutils literal notranslate"><span class="pre">batchIdx</span></code>, on gpu. In case of beam search, contains the ungathered data. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getGatheredIdsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a8301fa0adb3855c67e4e644f0725b3c5"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGatheredIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth, maxInputLength + maxNewTokens], only used for beam search. It contains gathered token ids without padding, on gpu. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getGatheredIds__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a43a18a77064d86372b206c78da31ed86"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGatheredIds</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>batchIdx</strong> – index of the batch </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>[batchSize, maxBeamWidth, maxInputLength + maxNewTokens], only used for beam search. It contains gathered token ids without padding for request <code class="docutils literal notranslate"><span class="pre">batchIdx</span></code>, on gpu. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState12getParentIdsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState12getParentIdsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState12getParentIdsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getParentIdsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a8c019b5cddff202635ea3e8a58026a22"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getParentIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState12getParentIdsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth, maxInputLength + maxNewTokens], contains parent ids collected during beam search without padding, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getCumLogProbsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ad0c0b44801ca328bc96184f69d9f6e87"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCumLogProbs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth], cumulative log probabilities (per beam), on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getCumLogProbs__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aff9385cdd976f23e079105c0bc68131e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCumLogProbs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[maxBeamWidth], cumulative log probabilities (per beam), on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getLogProbsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a66a9e1d24e63d83864294e723e1ed13f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogProbs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth, maxSequenceLength], log probabilities (per beam), on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getLogProbs__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a76275bef059fc7e2a24d7484d2e41222"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogProbs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[maxBeamWidth, maxSequenceLength], log probabilities (per beam), on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getSequenceLengthsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState18getSequenceLengthsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState18getSequenceLengthsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getSequenceLengthsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aec2b99d3378102e87c1f23089979fd9c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSequenceLengths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getSequenceLengthsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxBeamWidth], sequence lengths, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getAllNewTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState15getAllNewTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState15getAllNewTokensEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getAllNewTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a1313811f8c18a59d45a542374ee5f6df"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAllNewTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getAllNewTokensEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Get maxTokensPerStep tokens generated in the last forward pass. </p>
<dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[maxTokensPerStep, batchSize, maxBeamWidth], tokens generated in last forward pass, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getNextDraftTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState18getNextDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState18getNextDraftTokensEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getNextDraftTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a498d0defce0e90eb97542ae71c32142d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNextDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getNextDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxDraftTokens], predicted draft tokens for next step, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getPrevDraftTokensLengthsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState25getPrevDraftTokensLengthsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState25getPrevDraftTokensLengthsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getPrevDraftTokensLengthsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a1a2f301472d2a83b59d5f2d655ad718d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPrevDraftTokensLengths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getPrevDraftTokensLengthsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize], predicted draft tokens lengths for previous step, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getNextDraftTokensLengthsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState25getNextDraftTokensLengthsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState25getNextDraftTokensLengthsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getNextDraftTokensLengthsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ab0e476b820649c23847dcc701a6eaf88"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNextDraftTokensLengths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getNextDraftTokensLengthsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize], predicted draft tokens lengths for next step, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24getAcceptedLengthsCumSumEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState24getAcceptedLengthsCumSumEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState24getAcceptedLengthsCumSumEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getAcceptedLengthsCumSumC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a41a7031b75be3ee9599c10846ce15645"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAcceptedLengthsCumSum</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24getAcceptedLengthsCumSumEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize + 1], exclusive sum of accepted draft token lengths, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getAcceptedPackedPathsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState22getAcceptedPackedPathsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState22getAcceptedPackedPathsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getAcceptedPackedPathsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aa296b2014b5ec72a1e27a5facba68c81"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getAcceptedPackedPaths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getAcceptedPackedPathsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[batchSize, maxAcceptedDraftTokensPerStep], accepted paths packed into continuous tensor, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishedStepsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishedStepsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishedStepsEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getFinishedStepsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a44090220d7df07cc732b5b2db3649aea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFinishedSteps</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishedStepsEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>[maxTokensPerStep, batchSize, beamWidth], finished states of type FinishedState, on gpu </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getMaxBeamWidthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState15getMaxBeamWidthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState15getMaxBeamWidthEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getMaxBeamWidthC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1affb5c3e06a18f4e511a8f2662ed59013"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBeamWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getMaxBeamWidthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getMaxSequenceLengthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState20getMaxSequenceLengthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState20getMaxSequenceLengthEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getMaxSequenceLengthC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a4720903469a211026c5098beae8b6912"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxSequenceLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getMaxSequenceLengthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState27getMaxDecodingDecoderTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState27getMaxDecodingDecoderTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState27getMaxDecodingDecoderTokensEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getMaxDecodingDecoderTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aab5633fb93d667399cf4f36f2586b7b4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingDecoderTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState27getMaxDecodingDecoderTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getMaxDecodingEngineTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState26getMaxDecodingEngineTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState26getMaxDecodingEngineTokensEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getMaxDecodingEngineTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a4083420e98efb220f08a8f3fcb4c0c47"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingEngineTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getMaxDecodingEngineTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getNumDecodingEngineTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ade6ca9976b45e3eebbc26e04486a9d68"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getNumDecodingEngineTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Get the number of tokens for all requests in the batch. </p>
<dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>The number of tokens for all requests in the batch. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getNumDecodingEngineTokens__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a3f1315bfe74328b7be2fab4939c17657"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumDecodingEngineTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Get the number of tokens for a specific request in the batch. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>batchIdx</strong> – The index of the request in the batch. </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>The number of tokens for the specified request. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState26setNumDecodingEngineTokensE10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState26setNumDecodingEngineTokensE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState26setNumDecodingEngineTokensE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::setNumDecodingEngineTokens__SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ad940c6f656dee5d2243891e34209474b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumDecodingEngineTokens</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIdx</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numTokens</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState26setNumDecodingEngineTokensE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Set the number of tokens for a specific request in the batch. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>batchIdx</strong> – The index of the request in the batch. </p></li>
<li><p><strong>numTokens</strong> – The number of tokens for the specified request. </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getSpeculativeDecodingModeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState26getSpeculativeDecodingModeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState26getSpeculativeDecodingModeEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getSpeculativeDecodingModeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ae4cc9e8d67a255be108af23fec4a60bf"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpeculativeDecodingMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getSpeculativeDecodingModeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getBeamSearchBuffersEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState20getBeamSearchBuffersEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState20getBeamSearchBuffersEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getBeamSearchBuffersC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1adb726256c2898dc6eb2af559c6191dec"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE" title="tensorrt_llm::runtime::decoder::BeamSearchBuffers"><span class="n"><span class="pre">BeamSearchBuffers</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getBeamSearchBuffers</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getBeamSearchBuffersEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Workspace for beam search in streaming mode. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState21getJointDecodingInputEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState21getJointDecodingInputEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState21getJointDecodingInputEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getJointDecodingInputC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a4dfefcff30e619815aea4fbe5bd9eaca"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getJointDecodingInput</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState21getJointDecodingInputEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Stateful inputs for the decoder. Allocated for maxBatchSize slots. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getJointDecodingOutputEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7decoder12DecoderState22getJointDecodingOutputEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7decoder12DecoderState22getJointDecodingOutputEv"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::getJointDecodingOutputC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aa94cce5324b4ff78b7306f566d67936e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getJointDecodingOutput</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getJointDecodingOutputEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Stateful outputs for the decoder. Allocated for maxBatchSize slots. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBatchSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBatchSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBatchSizeE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mMaxBatchSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a8878bebf738d4102df98f0f18ecd5b9c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBatchSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBatchSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBeamWidthE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBeamWidthE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBeamWidthE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mMaxBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1aad69194fa07e93ff32c61affe53ef681"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBeamWidth</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBeamWidthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mMaxSequenceLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState18mMaxSequenceLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState18mMaxSequenceLengthE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mMaxSequenceLength__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1abbec68a2a7a4007c17a814a283d2c279"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxSequenceLength</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mMaxSequenceLengthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState19mJointDecodingInputE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState19mJointDecodingInputE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState19mJointDecodingInputE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mJointDecodingInput__DecodingInputPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a14e61d95d0643122a9e1095e9e0815a8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::DecodingInputPtr"><span class="n"><span class="pre">DecodingInputPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mJointDecodingInput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState19mJointDecodingInputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Stateful inputs for the decoder. Allocated for maxBatchSize slots. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState20mJointDecodingOutputE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState20mJointDecodingOutputE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState20mJointDecodingOutputE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mJointDecodingOutput__DecodingOutputPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a6af572c33736147efaceb70678b0f57c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::DecodingOutputPtr"><span class="n"><span class="pre">DecodingOutputPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mJointDecodingOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState20mJointDecodingOutputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Stateful outputs for the decoder. Allocated for maxBatchSize slots. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState14mFinishedStepsE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState14mFinishedStepsE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState14mFinishedStepsE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mFinishedSteps__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a5a9c0fec16681d733cf48d2bfafc9b9b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE" title="tensorrt_llm::runtime::decoder::DecoderState::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFinishedSteps</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState14mFinishedStepsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxTokensPerStep, batchSize, beamWidth] finished states of type FinishedState for each generated token of maxTokensPerStep, on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mBeamSearchBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState18mBeamSearchBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState18mBeamSearchBuffersE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mBeamSearchBuffers__std::unique_ptr:BeamSearchBuffers:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a1da6631ba958cc4f83e1fad3358de86a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE" title="tensorrt_llm::runtime::decoder::BeamSearchBuffers"><span class="n"><span class="pre">BeamSearchBuffers</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBeamSearchBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mBeamSearchBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><p>Workspace for beam search in streaming mode. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState25mMaxDecodingDecoderTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState25mMaxDecodingDecoderTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState25mMaxDecodingDecoderTokensE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mMaxDecodingDecoderTokens__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1ae20d729c46e47e339e1a4f04e7462779"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxDecodingDecoderTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState25mMaxDecodingDecoderTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mMaxDecodingEngineTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState24mMaxDecodingEngineTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState24mMaxDecodingEngineTokensE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mMaxDecodingEngineTokens__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a903dfe53d785cbc2fff847cf4847cb75"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxDecodingEngineTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mMaxDecodingEngineTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mNumDecodingEngineTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState24mNumDecodingEngineTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState24mNumDecodingEngineTokensE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mNumDecodingEngineTokens__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1a858f14b9613bee2f65adbfb1ed556ac1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumDecodingEngineTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mNumDecodingEngineTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize], the num tokens of each request. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mSpeculativeDecodingModeE">
<span id="_CPPv3N12tensorrt_llm7runtime7decoder12DecoderState24mSpeculativeDecodingModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime7decoder12DecoderState24mSpeculativeDecodingModeE"></span><span id="tensorrt_llm::runtime::decoder::DecoderState::mSpeculativeDecodingMode__SpeculativeDecodingMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder_1_1DecoderState_1af393cac2718ef81ac90be3628b680d6c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSpeculativeDecodingMode</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv" title="tensorrt_llm::runtime::SpeculativeDecodingMode::None"><span class="n"><span class="pre">None</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mSpeculativeDecodingModeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="request-h">
<h2>request.h<a class="headerlink" href="#request-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batchE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batchE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batchE"></span><span id="tensorrt_llm::runtime::decoder_batch"></span><span class="target" id="namespacetensorrt__llm_1_1runtime_1_1decoder__batch"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">decoder_batch</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="Link to this definition">#</a><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7RequestE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7RequestE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7RequestE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7RequestE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a6d5f16a0733fae53d2f6a74805cb012b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a95d9dd610e7a6295d06f75a33cbb4c43"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a78becbf7fa5928420a994c0d9d9ddb65"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request7RequestE14TensorConstPtr10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request7RequestE14TensorConstPtr10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request7RequestE14TensorConstPtr10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::Request__TensorConstPtr.SizeType32.std::optional:SizeType32:.std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a0dad449c01fb8b4af63de85659096fe0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Request</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">ids</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">inputLen</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxNewTokens</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">endId</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request7RequestE14TensorConstPtr10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request3idsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request3idsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request3idsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::ids__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a3e55abb4aecb8f67610629a44c74ae08"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ids</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request3idsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Mandatory parameters. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request8inputLenE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request8inputLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request8inputLenE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::inputLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a8762d6cd8c7a7efecb04d08c1d28e837"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inputLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request8inputLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12maxNewTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request12maxNewTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request12maxNewTokensE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::maxNewTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a5149bb633e0daa432cecc3d7c50483d3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxNewTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12maxNewTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5endIdE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request5endIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request5endIdE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::endId__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a9ab7466963ecf94768b4ecdc00702973"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">endId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5endIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request28generatedTokensPerEngineStepE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request28generatedTokensPerEngineStepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request28generatedTokensPerEngineStepE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::generatedTokensPerEngineStep__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1ad0961016791825df1e39e2d4c8bced7a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generatedTokensPerEngineStep</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request28generatedTokensPerEngineStepE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13embeddingBiasE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request13embeddingBiasE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request13embeddingBiasE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::embeddingBias__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1acbedf401f2f77534aa355f3ecbb0b755"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">embeddingBias</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13embeddingBiasE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12badWordsListE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request12badWordsListE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request12badWordsListE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::badWordsList__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1af18cc1700b613308c14d492e69ab76e9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">badWordsList</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12badWordsListE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13stopWordsListE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request13stopWordsListE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request13stopWordsListE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::stopWordsList__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1addd0a66af57e7186287ac7e5853c0c9d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stopWordsList</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13stopWordsListE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request11draftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request11draftTokensE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::draftTokens__BufferPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a1a524b5348e2d8ea442050f50ff3f1c7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::BufferPtr"><span class="n"><span class="pre">BufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>Optional parameters for speculative decoding. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request11draftLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request11draftLogitsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::draftLogits__std::optional:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1afa9181bc52e25fe7dc0e925c69dc8d86"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftLogitsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11medusaPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request11medusaPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request11medusaPathsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::medusaPaths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a2095b0e38974e5704d49f7b7226d1d62"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11medusaPathsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13medusaTreeIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request13medusaTreeIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request13medusaTreeIdsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::medusaTreeIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a53cd490ea4a4acc421b66a24ede31697"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Request::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaTreeIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13medusaTreeIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5dtypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request5dtypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request5dtypeE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::dtype__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a3b1b508a767e8f4af619a2553239319b"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">dtype</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5dtypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request22lookaheadRuntimeConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request22lookaheadRuntimeConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request22lookaheadRuntimeConfigE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::lookaheadRuntimeConfig__std::optional:executor::LookaheadDecodingConfig:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1a724413e68cfc7bea981a1b1b334a1704"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor23LookaheadDecodingConfigE" title="tensorrt_llm::executor::LookaheadDecodingConfig"><span class="n"><span class="pre">LookaheadDecodingConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lookaheadRuntimeConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request22lookaheadRuntimeConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11eagleConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch7Request11eagleConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch7Request11eagleConfigE"></span><span id="tensorrt_llm::runtime::decoder_batch::Request::eagleConfig__std::optional:executor::EagleConfig:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Request_1aaa297b6687699e8f43792997f503bef0"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor11EagleConfigE" title="tensorrt_llm::executor::EagleConfig"><span class="n"><span class="pre">EagleConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11eagleConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="loracache-h">
<h2>loraCache.h<a class="headerlink" href="#loracache-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9to_stringERKN9LoraCache21TaskLayerModuleConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime9to_stringERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime9to_stringERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="tensorrt_llm::runtime::to_string__LoraCache::TaskLayerModuleConfigCR"></span><span class="target" id="loraCache_8h_1ae685625093bdac0fb98a2ef13aa1a81e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">to_string</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">v</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9to_stringERKN9LoraCache21TaskLayerModuleConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN9LoraCache21TaskLayerModuleConfigE">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.LoraCache::TaskLayerModuleConfigCR"></span><span class="target" id="loraCache_8h_1a1455e26c47458506ad54c1140ac0776c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">v</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN9LoraCache21TaskLayerModuleConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCacheE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCacheE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCacheE"></span><span id="tensorrt_llm::runtime::LoraCache"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCache</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;loraCache.h&gt;</em></div>
<p><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCache"><span class="std std-ref">LoraCache</span></a></p>
<p>Caches LoRA weights with LRU eviction policy.</p>
<p>Tasks put in the cache are marked in progress and can not be evicted, until they are marked done.</p>
<p>A cache page holds a optimally sized LoRA. A page is of size [numSlots x pageWidth] An optimally size LoRA is on that has the configured optimalAdapterSize.</p>
<p>Conceptually a slot corresponds to a r=1, 1-layer, 1-module set of in/out weights. Page width is set to the number of weights in smallest module.</p>
<p>The number of slots per page is then ceilDiv(num weights in optimally sized LoRA, num weights in smallest module)</p>
<p>Cache pages are allocated on one or more blocks </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a52efeeb85da595056495553753dd5170"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a1db70e10833cee56b07d5d2c6a8895ef"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a806bd28154016a6d2f81ccdf676b3ebb"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskLayerModuleConfigListPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9LoraCacheERK26LoraCachePageManagerConfigRK11ModelConfigRK11WorldConfigRK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9LoraCacheERK26LoraCachePageManagerConfigRK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9LoraCacheERK26LoraCachePageManagerConfigRK11ModelConfigRK11WorldConfigRK13BufferManager"></span><span id="tensorrt_llm::runtime::LoraCache::LoraCache__LoraCachePageManagerConfigCR.ModelConfigCR.WorldConfigCR.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1ae8c4d09102e5af60a7b48f4657db3431"></span><span class="sig-name descname"><span class="n"><span class="pre">LoraCache</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pageManagerConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9LoraCacheERK26LoraCachePageManagerConfigRK11ModelConfigRK11WorldConfigRK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd><p>param[in] pageManagerConfig: a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig"><span class="std std-ref">LoraCachePageManagerConfig</span></a> param[in] modelConfig: a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ModelConfig"><span class="std std-ref">ModelConfig</span></a> param[in] worldConfig: a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1WorldConfig"><span class="std std-ref">WorldConfig</span></a> param[in] bufferManager: a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1BufferManager"><span class="std std-ref">BufferManager</span></a> only used to allocate page blocks </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache3putE10TaskIdType9TensorPtr9TensorPtrb">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache3putE10TaskIdType9TensorPtr9TensorPtrb"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache3putE10TaskIdType9TensorPtr9TensorPtrb"></span><span id="tensorrt_llm::runtime::LoraCache::put__TaskIdType.TensorPtr.TensorPtr.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a7a325af2ed20987b2248ec9155da6474"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">put</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">weights</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">load</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache3putE10TaskIdType9TensorPtr9TensorPtrb" title="Link to this definition">#</a><br /></dt>
<dd><p>put a task in the cache, and claim pages for it, and optionally load task weights. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p></li>
<li><p><strong>weights</strong> – <strong>[in]</strong> lora weights tensor </p></li>
<li><p><strong>config</strong> – <strong>[in]</strong> lora config tensor </p></li>
<li><p><strong>load</strong> – <strong>[in]</strong> if true load weights before returning, otherwise do not </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsE10TaskIdType9TensorPtr9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11loadWeightsE10TaskIdType9TensorPtr9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11loadWeightsE10TaskIdType9TensorPtr9TensorPtr"></span><span id="tensorrt_llm::runtime::LoraCache::loadWeights__TaskIdType.TensorPtr.TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a0153f9e24cf6ddd13aa2b33276b0d539"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loadWeights</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">weights</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsE10TaskIdType9TensorPtr9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd><p>load task weights. This method must be called after put. It is designed to be called asynchronously after put returns with load = false </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>taslId</strong> – <strong>[in]</strong> the task id </p></li>
<li><p><strong>weights</strong> – <strong>[in]</strong> lora weights tensor </p></li>
<li><p><strong>config</strong> – <strong>[in]</strong> lora config tensor </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache8isLoadedE10TaskIdType">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache8isLoadedE10TaskIdType"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache8isLoadedE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::isLoaded__TaskIdTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a0c328425bd4342282f2e948f62fa052b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLoaded</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache8isLoadedE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; true if task is loaded (weights are in place) and false otherwise </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache6isDoneE10TaskIdType">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache6isDoneE10TaskIdType"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache6isDoneE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::isDone__TaskIdTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a4ef43f6ccf206bb0c82a53f2d6474d0e"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isDone</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache6isDoneE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; true if task is marked done and can be evicted </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache3hasE10TaskIdType">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache3hasE10TaskIdType"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache3hasE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::has__TaskIdTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a8baa9098fbc9800417b3efea90f27f47"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">has</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache3hasE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; true if task is in the cache (not necessarily loaded) and false otherwise </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache3getE10TaskIdType">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache3getE10TaskIdType"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache3getE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::get__TaskIdType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a54af79acf17beb6ecc80dc9407ee087d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">get</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache3getE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; list of Value objects with pointers to task weights </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache4bumpE10TaskIdType">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache4bumpE10TaskIdType"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache4bumpE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::bump__TaskIdType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a48bc915d611d849249f459333c8a0ca6"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bump</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache4bumpE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><p>bump task and make it the most recently used </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache12markTaskDoneE10TaskIdType">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache12markTaskDoneE10TaskIdType"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache12markTaskDoneE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::markTaskDone__TaskIdType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1ac753d065a24dcee086f6d64825dc1249"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">markTaskDone</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12markTaskDoneE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><p>mark task done meaning it can be evicted </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the task id </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11markAllDoneEv">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11markAllDoneEv"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11markAllDoneEv"></span><span id="tensorrt_llm::runtime::LoraCache::markAllDone"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1ad6fb2ca1f0ab34248ebed9df893907d0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">markAllDone</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11markAllDoneEv" title="Link to this definition">#</a><br /></dt>
<dd><p>mark all tasks in cache done </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE10TaskIdType">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE10TaskIdType"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::determineNumPages__TaskIdTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a4690a3d893c213396d34fe1c4242dcf6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">determineNumPages</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>taskId</strong> – <strong>[in]</strong> the taskid </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; number of pages needed to store the given task </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE9TensorPtr">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE9TensorPtr"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE9TensorPtr"></span><span id="tensorrt_llm::runtime::LoraCache::determineNumPages__TensorPtrC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1af8e36fb16dcc5ecb114b6bd1e4f89b60"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">determineNumPages</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>config</strong> – <strong>[in]</strong> lora config tensor </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; number of pages needed to store the task configured with config tensor </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache4fitsE9TensorPtr">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache4fitsE9TensorPtr"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache4fitsE9TensorPtr"></span><span id="tensorrt_llm::runtime::LoraCache::fits__TensorPtrC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1aee17e54b1070ffca531aaf6ff9fdfdef"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fits</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache4fitsE9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>config</strong> – <strong>[in]</strong> a lora config tensor </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; true in task fits in cache false otherwise </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache8copyTaskE10TaskIdTypeR9LoraCacheb">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache8copyTaskE10TaskIdTypeR9LoraCacheb"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache8copyTaskE10TaskIdTypeR9LoraCacheb"></span><span id="tensorrt_llm::runtime::LoraCache::copyTask__TaskIdType.LoraCacheR.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a3fe38aa85a89b76a354cca719b65d39b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyTask</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">deviceCache</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">markDone</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache8copyTaskE10TaskIdTypeR9LoraCacheb" title="Link to this definition">#</a><br /></dt>
<dd><p>copy task to another cache. Caches must have the same page size. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>taskId</strong> – <strong>[in]</strong> the task id to copy </p></li>
<li><p><strong>otherCache</strong> – <strong>[in]</strong> the <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCache"><span class="std std-ref">LoraCache</span></a> to move the task to </p></li>
<li><p><strong>markDone</strong> – <strong>[in]</strong> mark the copied task done as it’s copied </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache11getNumPagesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache11getNumPagesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache11getNumPagesEv"></span><span id="tensorrt_llm::runtime::LoraCache::getNumPagesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1adcd24dca8d65f4862021ae2fa84b9199"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumPages</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache11getNumPagesEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>&amp;#8212; total number of pages allocated to cache (used or not) </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache10getPagePtrE6size_t">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache10getPagePtrE6size_t"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache10getPagePtrE6size_t"></span><span id="tensorrt_llm::runtime::LoraCache::getPagePtr__sC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1af4191f7de099ddabb12699b7c7613812"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPagePtr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pageId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache10getPagePtrE6size_t" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>pageId</strong> – <strong>[in]</strong> the page id </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; const pointer to page </p>
</dd>
</dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11copyToPagesE9TensorPtr9TensorPtrRK11ModelConfigRK11WorldConfigNSt13unordered_mapI10SizeType3210LoraModuleEERK13BufferManagerRKNSt6vectorI9TensorPtrEERKNSt6vectorINSt6size_tEEE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11copyToPagesE9TensorPtr9TensorPtrRK11ModelConfigRK11WorldConfigNSt13unordered_mapI10SizeType3210LoraModuleEERK13BufferManagerRKNSt6vectorI9TensorPtrEERKNSt6vectorINSt6size_tEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11copyToPagesE9TensorPtr9TensorPtrRK11ModelConfigRK11WorldConfigNSt13unordered_mapI10SizeType3210LoraModuleEERK13BufferManagerRKNSt6vectorI9TensorPtrEERKNSt6vectorINSt6size_tEEE"></span><span id="tensorrt_llm::runtime::LoraCache::copyToPages__TensorPtr.TensorPtr.ModelConfigCR.WorldConfigCR.std::unordered_map:SizeType32.LoraModule:.BufferManagerCR.std::vector:TensorPtr:CR.std::vector:std::s:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1ac67eb5e9fdcb057e560bdb2e62e89ef5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyToPages</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">weights</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">moduleIdToModel</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pages</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pageIds</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11copyToPagesE9TensorPtr9TensorPtrRK11ModelConfigRK11WorldConfigNSt13unordered_mapI10SizeType3210LoraModuleEERK13BufferManagerRKNSt6vectorI9TensorPtrEERKNSt6vectorINSt6size_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy task weights to cache pages. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>weights</strong> – <strong>[in]</strong> task weights </p></li>
<li><p><strong>config</strong> – <strong>[in]</strong> task config tensor </p></li>
<li><p><strong>modelConfig</strong> – <strong>[in]</strong> a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ModelConfig"><span class="std std-ref">ModelConfig</span></a></p></li>
<li><p><strong>worldConfig</strong> – <strong>[in]</strong> a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1WorldConfig"><span class="std std-ref">WorldConfig</span></a></p></li>
<li><p><strong>modelIdToModel</strong> – <strong>[in]</strong> map from lora module id to <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraModule"><span class="std std-ref">LoraModule</span></a></p></li>
<li><p><strong>manager</strong> – <strong>[in]</strong> a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1BufferManager"><span class="std std-ref">BufferManager</span></a> the manager to use to perform the copies </p></li>
<li><p><strong>pages</strong> – <strong>[out]</strong> list of page tensors to copy weights to </p></li>
<li><p><strong>pageIds</strong> – <strong>[in]</strong> page ids for the pages </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; list of cache Values objects </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache17splitTransposeCpuER7ITensorRK7ITensor10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache17splitTransposeCpuER7ITensorRK7ITensor10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache17splitTransposeCpuER7ITensorRK7ITensor10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraCache::splitTransposeCpu__ITensorR.ITensorCR.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a2fc01315185a952f749316961be48494"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">splitTransposeCpu</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpRank</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17splitTransposeCpuER7ITensorRK7ITensor10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>splits second dim of input into tpSize parts and writes the tpRank split to output </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>output</strong> – <strong>[out]</strong> output tensor </p></li>
<li><p><strong>input</strong> – <strong>[in]</strong> input tensor </p></li>
<li><p><strong>tpSize</strong> – <strong>[in]</strong> number of splits </p></li>
<li><p><strong>tpRank</strong> – <strong>[in]</strong> the split to write to output </p></li>
</ul>
</dd>
</dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
<dl class="cpp enum">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatusE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11ValueStatusE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11ValueStatusE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a75949c4322be311f8f485c1b3983a4a8"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ValueStatus</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatusE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus21kVALUE_STATUS_MISSINGE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11ValueStatus21kVALUE_STATUS_MISSINGE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11ValueStatus21kVALUE_STATUS_MISSINGE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a75949c4322be311f8f485c1b3983a4a8a71919c7fd85566514a9abcd3e631e892"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kVALUE_STATUS_MISSING</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus21kVALUE_STATUS_MISSINGE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus24kVALUE_STATUS_PROCESSINGE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11ValueStatus24kVALUE_STATUS_PROCESSINGE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11ValueStatus24kVALUE_STATUS_PROCESSINGE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a75949c4322be311f8f485c1b3983a4a8a7693ca37a375c6f160c7608ecf999617"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kVALUE_STATUS_PROCESSING</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus24kVALUE_STATUS_PROCESSINGE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus20kVALUE_STATUS_LOADEDE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11ValueStatus20kVALUE_STATUS_LOADEDE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11ValueStatus20kVALUE_STATUS_LOADEDE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a75949c4322be311f8f485c1b3983a4a8a599f5d302d900314903fd3ab8c2be0d1"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kVALUE_STATUS_LOADED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus20kVALUE_STATUS_LOADEDE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1af49013c48c41ec7b89e862a35f36bbee"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskValuePtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsER9TaskValue9TensorPtr9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11loadWeightsER9TaskValue9TensorPtr9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11loadWeightsER9TaskValue9TensorPtr9TensorPtr"></span><span id="tensorrt_llm::runtime::LoraCache::loadWeights__TaskValueR.TensorPtr.TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a0264725d06825eee042d00c976c20cf6"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loadWeights</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">cacheValue</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">weights</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE" title="tensorrt_llm::runtime::LoraCache::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsER9TaskValue9TensorPtr9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache18bumpTaskInProgressE10TaskIdType">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache18bumpTaskInProgressE10TaskIdType"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache18bumpTaskInProgressE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::bumpTaskInProgress__TaskIdType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a776bdc112fb5c1d966d83d8c5fbe0f40"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bumpTaskInProgress</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache18bumpTaskInProgressE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache9getStatusE10TaskIdType">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache9getStatusE10TaskIdType"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache9getStatusE10TaskIdType"></span><span id="tensorrt_llm::runtime::LoraCache::getStatus__TaskIdTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1aa0f3c256a9ebc2d30465a6a9e490fb42"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatusE" title="tensorrt_llm::runtime::LoraCache::ValueStatus"><span class="n"><span class="pre">ValueStatus</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getStatus</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">taskId</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache9getStatusE10TaskIdType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache19claimPagesWithEvictE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache19claimPagesWithEvictE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache19claimPagesWithEvictE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCache::claimPagesWithEvict__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a42cd991f5baeba5c72aff9d90fe2932d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">claimPagesWithEvict</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numPages</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache19claimPagesWithEvictE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>claim numPages, evicting tasks if needed </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>numPages</strong> – <strong>[in]</strong> number of pages to claim </p>
</dd>
<dt class="field-even">Throws<span class="colon">:</span></dt>
<dd class="field-even"><p><span><span class="cpp-expr sig sig-inline cpp"><span class="n">std</span><span class="p">::</span><span class="n">runtime_error</span></span></span> – if all pages cannot be claimed </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>&amp;#8212; list of page ids </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache16copyTaskMapPagesER9TaskValueRK9TaskValueRKNSt6vectorI6size_tEERK9LoraCache">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache16copyTaskMapPagesER9TaskValueRK9TaskValueRKNSt6vectorI6size_tEERK9LoraCache"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache16copyTaskMapPagesER9TaskValueRK9TaskValueRKNSt6vectorI6size_tEERK9LoraCache"></span><span id="tensorrt_llm::runtime::LoraCache::copyTaskMapPages__TaskValueR.TaskValueCR.std::vector:s:CR.LoraCacheCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a3cd51bc551695aade65b9683f2ed622c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyTaskMapPages</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">targetTaskValue</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">sourceTaskValue</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">targetPageIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">targetCache</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache16copyTaskMapPagesER9TaskValueRK9TaskValueRKNSt6vectorI6size_tEERK9LoraCache" title="Link to this definition">#</a><br /></dt>
<dd><p>Internal helper method used inside copyTask. Not thread safe on its own </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache18mPageManagerConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache18mPageManagerConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache18mPageManagerConfigE"></span><span id="tensorrt_llm::runtime::LoraCache::mPageManagerConfig__LoraCachePageManagerConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a48ca80214b9115983030b1b406c1e4fc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPageManagerConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache18mPageManagerConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache12mModelConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache12mModelConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache12mModelConfigE"></span><span id="tensorrt_llm::runtime::LoraCache::mModelConfig__ModelConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a0064ca2bd7e388ef4766534041f0d867"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12mModelConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache12mWorldConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache12mWorldConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache12mWorldConfigE"></span><span id="tensorrt_llm::runtime::LoraCache::mWorldConfig__WorldConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a85f9991a8aa4f25db6d6b3505474f6c9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mWorldConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12mWorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11mPagesMutexE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11mPagesMutexE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11mPagesMutexE"></span><span id="tensorrt_llm::runtime::LoraCache::mPagesMutex__std::mutex"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1acced4dd499010314b3ba5c6661e1f4cd"></span><span class="k"><span class="pre">mutable</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">mutex</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPagesMutex</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11mPagesMutexE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache17mCachePageManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache17mCachePageManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache17mCachePageManagerE"></span><span id="tensorrt_llm::runtime::LoraCache::mCachePageManager__std::unique_ptr:LoraCachePageManager:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a7a349c6ef0735c024bd59e60f3a2ff62"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManagerE" title="tensorrt_llm::runtime::LoraCachePageManager"><span class="n"><span class="pre">LoraCachePageManager</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCachePageManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17mCachePageManagerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache11mCacheMutexE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache11mCacheMutexE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache11mCacheMutexE"></span><span id="tensorrt_llm::runtime::LoraCache::mCacheMutex__std::mutex"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1ad05d0ba4aa25bffef6f40d88119e4a4b"></span><span class="k"><span class="pre">mutable</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">mutex</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCacheMutex</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11mCacheMutexE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9mCacheMapE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9mCacheMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9mCacheMapE"></span><span id="tensorrt_llm::runtime::LoraCache::mCacheMap__std::unordered_map:TaskIdType.TaskValuePtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a925da94c9416d9e1206c5a79bd772c60"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE" title="tensorrt_llm::runtime::LoraCache::TaskValuePtr"><span class="n"><span class="pre">TaskValuePtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCacheMap</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9mCacheMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache16mInProgressTasksE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache16mInProgressTasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache16mInProgressTasksE"></span><span id="tensorrt_llm::runtime::LoraCache::mInProgressTasks__std::list:TaskIdType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a0f51387f4a795d072b2c53389c2ad0e7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInProgressTasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache16mInProgressTasksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache10mDoneTasksE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache10mDoneTasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache10mDoneTasksE"></span><span id="tensorrt_llm::runtime::LoraCache::mDoneTasks__std::list:TaskIdType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a7786c6df0ce94dc5cbc7f877d989c932"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDoneTasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10mDoneTasksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21mDeviceBufferManagersE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21mDeviceBufferManagersE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21mDeviceBufferManagersE"></span><span id="tensorrt_llm::runtime::LoraCache::mDeviceBufferManagers__std::vector:std::unique_ptr:BufferManager::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1aaee288e322ff0875437dc466b889b605"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDeviceBufferManagers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21mDeviceBufferManagersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache14mBufferManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache14mBufferManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache14mBufferManagerE"></span><span id="tensorrt_llm::runtime::LoraCache::mBufferManager__std::unique_ptr:BufferManager:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a3b33e7cb854d144ec9f62c0724dcfeb1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBufferManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache14mBufferManagerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache17mModuleIdToModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache17mModuleIdToModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache17mModuleIdToModuleE"></span><span id="tensorrt_llm::runtime::LoraCache::mModuleIdToModule__std::unordered_map:SizeType32.LoraModule:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a57ff4ff69308b55880fc4460dd1d4a4f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModuleIdToModule</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17mModuleIdToModuleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime9LoraCache22splitTransposeCpuInnerEvR7ITensorRK7ITensor10SizeType3210SizeType32">
<span id="_CPPv3I0EN12tensorrt_llm7runtime9LoraCache22splitTransposeCpuInnerER7ITensorRK7ITensor10SizeType3210SizeType32"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime9LoraCache22splitTransposeCpuInnerER7ITensorRK7ITensor10SizeType3210SizeType32"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCache_1a5ebe7179a3661f5e3ada638cd0fc578a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">splitTransposeCpuInner</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpRank</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime9LoraCache22splitTransposeCpuInnerEvR7ITensorRK7ITensor10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;loraCache.h&gt;</em></div>
<p>Contains information on a single layer / module. A list of these configs is associated with each task and can be used to populate runtime tensors. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8toStringEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8toStringEv"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::toStringC"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a22d58d87f683ff4c3c110bcea6ac76cf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8toStringEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigeqERKN9LoraCache21TaskLayerModuleConfigE">
<span id="_CPPv3NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigeqERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="_CPPv2NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigeqERKN9LoraCache21TaskLayerModuleConfigE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::eq-operator__LoraCache::TaskLayerModuleConfigCRC"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1aeeb90d000b1ebe5532ec51e9012b3d73"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE" title="tensorrt_llm::runtime::LoraCache"><span class="n"><span class="pre">LoraCache</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig"><span class="n"><span class="pre">TaskLayerModuleConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigeqERKN9LoraCache21TaskLayerModuleConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6pageIdE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6pageIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6pageIdE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::pageId__std::s"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1acb73dea546a2d4be0143cb0f01a81c15"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pageId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6pageIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7slotIdxE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7slotIdxE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7slotIdxE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::slotIdx__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a7ab99965797b2b31e63b66bab9c7dd9c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slotIdx</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7slotIdxE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6inSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6inSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6inSizeE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::inSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a2cbe1e0f50a01f3d781216c627c6b0ba"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6inSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7outSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7outSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7outSizeE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::outSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a2ff87ee039d5c46915d981c11114ddf0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7outSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8moduleIdE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8moduleIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8moduleIdE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::moduleId__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1ab0a616711dea88ac63fe31043699b0b9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">moduleId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8moduleIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7layerIdE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7layerIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7layerIdE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::layerId__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a02012816d82515dd66bad77eb9e0ea26"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">layerId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7layerIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig11adapterSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig11adapterSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig11adapterSizeE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::adapterSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1af91dca0e1a714f02cd9d62979ef78463"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">adapterSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig11adapterSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8numSlotsE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8numSlotsE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8numSlotsE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::numSlots__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a0b6d002a8a14a64f26b54152b614ab3b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8numSlotsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig16weightsInPointerE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig16weightsInPointerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig16weightsInPointerE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::weightsInPointer__std::int64_t"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1a67ab9af3fd4782431150b26241f49cdc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">weightsInPointer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig16weightsInPointerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17weightsOutPointerE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17weightsOutPointerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17weightsOutPointerE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::weightsOutPointer__std::int64_t"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1aa5883002221d6f5771219423d5aadb14"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">weightsOutPointer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17weightsOutPointerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17scalingVecPointerE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17scalingVecPointerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17scalingVecPointerE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig::scalingVecPointer__std::optional:std::int64_t:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskLayerModuleConfig_1af64e88f2f5e17ff07092b922a6645100"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scalingVecPointer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17scalingVecPointerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
<dl>
<dt class="sig sig-object cpp">
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">TaskLayerModuleConfigBindings</span></dt>
</dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValueE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValueE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskValue</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="Link to this definition">#</a><br /></dt>
<dd><p>Holds configuration and state for a single task. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueEv">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueEv"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueEv"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::TaskValue"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1ac50c3976697f9adbf75af8f5ff4398ad"></span><span class="sig-name descname"><span class="n"><span class="pre">TaskValue</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValueD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValueD0Ev"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::~TaskValue"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a03e4986e9975fe5f53eae770fc730611"></span><span class="sig-name descname"><span class="n"><span class="pre">~TaskValue</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERKNSt6vectorINSt6size_tEEERK28TaskLayerModuleConfigListPtrNSt4listI10TaskIdTypeE8iteratorEbbbb">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERKNSt6vectorINSt6size_tEEERK28TaskLayerModuleConfigListPtrNSt4listI10TaskIdTypeE8iteratorEbbbb"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERKNSt6vectorINSt6size_tEEERK28TaskLayerModuleConfigListPtrNSt4listI10TaskIdTypeE8iteratorEbbbb"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::TaskValue__std::vector:std::s:CR.TaskLayerModuleConfigListPtrCR.std::list:TaskIdType:::iterator.b.b.b.b"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a57e05d4fc10716c2ace0d4cd5373be2a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskValue</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pageIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfigListPtr"><span class="n"><span class="pre">TaskLayerModuleConfigListPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">configs</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">iterator</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">it</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inProgress</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">loaded</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">done</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">loadInProgress</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERKNSt6vectorINSt6size_tEEERK28TaskLayerModuleConfigListPtrNSt4listI10TaskIdTypeE8iteratorEbbbb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::TaskValue__TaskValueRR"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a5691f27c3933372b45897851e372830e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TaskValue</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue" title="tensorrt_llm::runtime::LoraCache::TaskValue::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueaSERR9TaskValue">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValueaSERR9TaskValue"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValueaSERR9TaskValue"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::assign-operator__TaskValueRR"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a347d57780454a36ce09ea22e211a08d9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE" title="tensorrt_llm::runtime::LoraCache::TaskValue"><span class="n"><span class="pre">TaskValue</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueaSERR9TaskValue" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7pageIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue7pageIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue7pageIdsE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::pageIds__std::vector:std::s:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a374f2e22fe8185f13d0900cba0930828"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pageIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7pageIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7configsE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue7configsE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue7configsE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::configs__TaskLayerModuleConfigListPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a55350722530644cc428078c07202defc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE" title="tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfigListPtr"><span class="n"><span class="pre">TaskLayerModuleConfigListPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">configs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7configsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue2itE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue2itE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue2itE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::it__std::list:TaskIdType:::iterator"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a86d754491bf621bea52a641aa7f2c05a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE" title="tensorrt_llm::runtime::LoraCache::TaskIdType"><span class="n"><span class="pre">TaskIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">iterator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">it</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue2itE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue10inProgressE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue10inProgressE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue10inProgressE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::inProgress__b"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a75db6091b5e8ffe849dcb99f0b50c383"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inProgress</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue10inProgressE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue6loadedE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue6loadedE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue6loadedE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::loaded__b"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a3f073e2982f8d32b58fd2836ac1cea96"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loaded</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue6loadedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue4doneE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue4doneE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue4doneE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::done__b"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1aab94235b5a3c2d5ab7187bbbe334d57e"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">done</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue4doneE" title="Link to this definition">#</a><br /></dt>
<dd><p>Marks a task a done. This is used to mark a task as done during loading. if done=true at the end of loading (end of put, loadweights, or copyTask) the task will be marked as done </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue14loadInProgressE">
<span id="_CPPv3N12tensorrt_llm7runtime9LoraCache9TaskValue14loadInProgressE"></span><span id="_CPPv2N12tensorrt_llm7runtime9LoraCache9TaskValue14loadInProgressE"></span><span id="tensorrt_llm::runtime::LoraCache::TaskValue::loadInProgress__b"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1LoraCache_1_1TaskValue_1a98f1c9b6a5fe00e9009b366bf3861a17"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">loadInProgress</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue14loadInProgressE" title="Link to this definition">#</a><br /></dt>
<dd><p>Indicates weights are loading either in put or loadWeights This is used to block concurrent loadWeights calls for the same task. </p>
</dd></dl>
</div>
</dd></dl>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionE">
<span id="_CPPv3N12tensorrt_llm7runtime22LoraCacheFullExceptionE"></span><span id="_CPPv2N12tensorrt_llm7runtime22LoraCacheFullExceptionE"></span><span id="tensorrt_llm::runtime::LoraCacheFullException"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCacheFullException"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCacheFullException</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionE" title="tensorrt_llm::runtime::LoraExpectedException"><span class="n"><span class="pre">LoraExpectedException</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime22LoraCacheFullException22LoraCacheFullExceptionERKNSt6stringE">
<span id="_CPPv3N12tensorrt_llm7runtime22LoraCacheFullException22LoraCacheFullExceptionERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm7runtime22LoraCacheFullException22LoraCacheFullExceptionERKNSt6stringE"></span><span id="tensorrt_llm::runtime::LoraCacheFullException::LoraCacheFullException__ssCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCacheFullException_1aefa74dd5535c638a6b5d16d886db0887"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCacheFullException</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">msg</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullException22LoraCacheFullExceptionERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime22LoraCacheFullExceptionD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime22LoraCacheFullExceptionD0Ev"></span><span id="tensorrt_llm::runtime::LoraCacheFullException::~LoraCacheFullException"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCacheFullException_1a7ab9608608f16e2186d8fedd599b1c26"></span><span class="sig-name descname"><span class="n"><span class="pre">~LoraCacheFullException</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManagerE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCachePageManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManagerE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;loraCache.h&gt;</em></div>
<p>Holds memory of lora cache pages, and manages allocation and freeing of whole pages. Memory is pre-allocated either on the host or device</p>
<p>Note that this class is not thread safe </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a8ddec7e4c764aefb6ca1e7983f10f6c6"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager20LoraCachePageManagerERK26LoraCachePageManagerConfigRK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager20LoraCachePageManagerERK26LoraCachePageManagerConfigRK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager20LoraCachePageManagerERK26LoraCachePageManagerConfigRK13BufferManager"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::LoraCachePageManager__LoraCachePageManagerConfigCR.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a4f03d18ae0cb624f74e470baec8e4cd4"></span><span class="sig-name descname"><span class="n"><span class="pre">LoraCachePageManager</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">config</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager20LoraCachePageManagerERK26LoraCachePageManagerConfigRK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>config</strong> – <strong>[in]</strong> a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig"><span class="std std-ref">LoraCachePageManagerConfig</span></a></p></li>
<li><p><strong>bufferManager</strong> – <strong>[in]</strong> a Buffermanager used to allocate page blocks </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10claimPagesE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager10claimPagesE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager10claimPagesE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::claimPages__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1aa931d613427ce9df18abdac708fcec01"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">claimPages</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numPages</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10claimPagesE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>claim pages </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>numPages</strong> – <strong>[in]</strong> number of pages to claim </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>a tuple, where the first values is a boolean indicating whether pages were claimed. If the first value is true the second value will have a list of pageIds </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager17numAvailablePagesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime20LoraCachePageManager17numAvailablePagesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime20LoraCachePageManager17numAvailablePagesEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::numAvailablePagesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1ab17136bfabc750108a3ab318e60d5d56"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numAvailablePages</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager17numAvailablePagesEv" title="Link to this definition">#</a><br /></dt>
<dd><p>get number of available (free) pages in manager </p>
<dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>number of free pages in manager </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12releasePagesERKNSt6vectorINSt6size_tEEE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager12releasePagesERKNSt6vectorINSt6size_tEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager12releasePagesERKNSt6vectorINSt6size_tEEE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::releasePages__std::vector:std::s:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1afc927205186a82307fc039f95ce2e0e1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">releasePages</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pages</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12releasePagesERKNSt6vectorINSt6size_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>release given pages </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>pages</strong> – <strong>[in]</strong> list of pages to release (free) </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager8blockPtrE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime20LoraCachePageManager8blockPtrE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime20LoraCachePageManager8blockPtrE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::blockPtr__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a61181364d46c445940a2940e6ca3d469"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">blockPtr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">blockIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager8blockPtrE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>return pointer to given page block </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>blockIdx;</strong> – <strong>[in]</strong> </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; pointer to page block </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager7pagePtrENSt6size_tE">
<span id="_CPPv3NK12tensorrt_llm7runtime20LoraCachePageManager7pagePtrENSt6size_tE"></span><span id="_CPPv2NK12tensorrt_llm7runtime20LoraCachePageManager7pagePtrENSt6size_tE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::pagePtr__std::sC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a1a30baf3b114912fccbcdf2ad649755a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pagePtr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pageIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager7pagePtrENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>return pointer to given page </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>pageIdx</strong> – <strong>[in]</strong> </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; const pointer to page </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager14mutablePagePtrENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager14mutablePagePtrENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager14mutablePagePtrENSt6size_tE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::mutablePagePtr__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a8e52eebebbe95fb739988456aded1b2a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mutablePagePtr</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pageIdx</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager14mutablePagePtrENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>return pointer to given page </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>pageIdx</strong> – <strong>[in]</strong> </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>&amp;#8212; mutable pointer to page </p>
</dd>
</dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10initializeERK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager10initializeERK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager10initializeERK13BufferManager"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::initialize__BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a66be850fa0c5232ea5cbc330bc363b5a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">initialize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10initializeERK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mPageBlocksE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager11mPageBlocksE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager11mPageBlocksE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::mPageBlocks__std::vector:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1af4e201033f3d2b283a526e204d6f9731"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE" title="tensorrt_llm::runtime::LoraCachePageManager::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPageBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mPageBlocksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12mFreePageIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager12mFreePageIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager12mFreePageIdsE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::mFreePageIds__std::deque:std::s:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a57cfac298b65e0a2d6c539d7aebdb85c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">deque</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mFreePageIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12mFreePageIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mIsPageFreeE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager11mIsPageFreeE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager11mIsPageFreeE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::mIsPageFree__std::vector:std::uint8_t:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1a8339dcb267e8b1949248dd4567684cd2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIsPageFree</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mIsPageFreeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager7mConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime20LoraCachePageManager7mConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime20LoraCachePageManager7mConfigE"></span><span id="tensorrt_llm::runtime::LoraCachePageManager::mConfig__LoraCachePageManagerConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManager_1af90e90b30a4712ff3dee148f986e6332"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager7mConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionE">
<span id="_CPPv3N12tensorrt_llm7runtime21LoraExpectedExceptionE"></span><span id="_CPPv2N12tensorrt_llm7runtime21LoraExpectedExceptionE"></span><span id="tensorrt_llm::runtime::LoraExpectedException"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraExpectedException"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraExpectedException</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">runtime_error</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionE" title="Link to this definition">#</a><br /></dt>
<dd><p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCacheFullException"><span class="std std-ref">tensorrt_llm::runtime::LoraCacheFullException</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime21LoraExpectedException21LoraExpectedExceptionERKNSt6stringE">
<span id="_CPPv3N12tensorrt_llm7runtime21LoraExpectedException21LoraExpectedExceptionERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm7runtime21LoraExpectedException21LoraExpectedExceptionERKNSt6stringE"></span><span id="tensorrt_llm::runtime::LoraExpectedException::LoraExpectedException__ssCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraExpectedException_1a2e41970c7453e194635c9c7ca583bd99"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraExpectedException</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">msg</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedException21LoraExpectedExceptionERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime21LoraExpectedExceptionD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime21LoraExpectedExceptionD0Ev"></span><span id="tensorrt_llm::runtime::LoraExpectedException::~LoraExpectedException"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraExpectedException_1a84b390f6dbb652b8501868df858aab64"></span><span class="sig-name descname"><span class="n"><span class="pre">~LoraExpectedException</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="buffermanager-h">
<h2>bufferManager.h<a class="headerlink" href="#buffermanager-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManagerE"></span><span id="tensorrt_llm::runtime::BufferManager"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;bufferManager.h&gt;</em></div>
<p>A helper class for managing memory on host and device. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager10IBufferPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager10IBufferPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a0bd5b422b1e3a5d2171f66532f27f413"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IBufferPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager10ITensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager10ITensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a5f8cccf8f635cf3cfbec353e1a7317b9"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ITensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1ae977afd5a423417ac57f3bffa8162b33"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a116255e34fbfe37cb623ef2e280eb9fc"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaMemPoolPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">CudaMemPool</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager13BufferManagerE13CudaStreamPtrb">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager13BufferManagerE13CudaStreamPtrb"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager13BufferManagerE13CudaStreamPtrb"></span><span id="tensorrt_llm::runtime::BufferManager::BufferManager__CudaStreamPtr.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a1736c6a4c4a327a00a117addf7cfd915"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferManager</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE" title="tensorrt_llm::runtime::BufferManager::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">trimPool</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13BufferManagerE13CudaStreamPtrb" title="Link to this definition">#</a><br /></dt>
<dd><p>Construct a <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1BufferManager"><span class="std std-ref">BufferManager</span></a>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>cudaStream</strong> – <strong>[in]</strong> The cuda stream to use for all operations on GPU (allocation, de-allocation, copying, etc.). </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManagerD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManagerD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManagerD0Ev"></span><span id="tensorrt_llm::runtime::BufferManager::~BufferManager"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1aea6ac13251be9f98419b4b20d9ee5b50"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~BufferManager</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerD0Ev" title="Link to this definition">#</a><br /></dt>
<dd><p>Destructor. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager3gpuENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager3gpuENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::gpu__std::s.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a2cf0e00fd6ac0cfe8b593749583a7a03"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpu</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size on the GPU, using cudaMallocAsync. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager3gpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager3gpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::gpu__nvinfer1::Dims.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a5a023b2d5f629bc86e262f9733a12a59"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpu</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions on the GPU, using cudaMallocAsync. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeNSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeNSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeNSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::allocate__MemoryType.std::s.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a1bdafec3fa633fdb0d321f5e748b3174"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocate</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeNSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size and memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::allocate__MemoryType.nvinfer1::Dims.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a0e4e4c0e248e95e19061a61fadec30cc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocate</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions and memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyBufferE10MemoryTypeN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager11emptyBufferE10MemoryTypeN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager11emptyBufferE10MemoryTypeN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::emptyBuffer__MemoryType.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a0e78e3aa2aa1d635ae230996b449f9a4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">emptyBuffer</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyBufferE10MemoryTypeN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Create an empty <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given memory type. It may be resized later. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyTensorE10MemoryTypeN8nvinfer18DataTypeE">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager11emptyTensorE10MemoryTypeN8nvinfer18DataTypeE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager11emptyTensorE10MemoryTypeN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::emptyTensor__MemoryType.nvinfer1::DataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a1d0157fa19df03ba561d15d5b64de259"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">emptyTensor</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyTensorE10MemoryTypeN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Create an empty <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given memory type. It may be reshaped later. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager6setMemER7IBuffer7int32_t">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager6setMemER7IBuffer7int32_t"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager6setMemER7IBuffer7int32_t"></span><span id="tensorrt_llm::runtime::BufferManager::setMem__IBufferR.int32_tC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1ab625f396732dddcad22f44e9ab02fc28"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMem</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">int32_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">value</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager6setMemER7IBuffer7int32_t" title="Link to this definition">#</a><br /></dt>
<dd><p>Set the contents of the given <code class="docutils literal notranslate"><span class="pre">buffer</span></code> to value. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager7setZeroER7IBuffer">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager7setZeroER7IBuffer"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager7setZeroER7IBuffer"></span><span id="tensorrt_llm::runtime::BufferManager::setZero__IBufferRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1afe66c31bfc8e00ebbdb8405194be0db9"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setZero</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager7setZeroER7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Set the contents of the given <code class="docutils literal notranslate"><span class="pre">buffer</span></code> to zero. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer10MemoryType">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer10MemoryType"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer10MemoryType"></span><span id="tensorrt_llm::runtime::BufferManager::copy__voidCP.IBufferR.MemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a1adc12c1388e8a3392ac39a3ae64d518"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copy</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dst</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">srcType</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> to <code class="docutils literal notranslate"><span class="pre">dst</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv10MemoryType">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv10MemoryType"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv10MemoryType"></span><span id="tensorrt_llm::runtime::BufferManager::copy__IBufferCR.voidP.MemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a313f2bd39b26e568c97f092a98ac20f9"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copy</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">dst</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dstType</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> to <code class="docutils literal notranslate"><span class="pre">dst</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer"></span><span id="tensorrt_llm::runtime::BufferManager::copy__voidCP.IBufferRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a3086524a384108c45199076f536d4d9c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copy</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dst</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> to <code class="docutils literal notranslate"><span class="pre">dst</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv"></span><span id="tensorrt_llm::runtime::BufferManager::copy__IBufferCR.voidPC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a583c0a02dcd51050b5b4273a19c0bde7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copy</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">dst</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> to <code class="docutils literal notranslate"><span class="pre">dst</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferR7IBuffer">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferR7IBuffer"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferR7IBuffer"></span><span id="tensorrt_llm::runtime::BufferManager::copy__IBufferCR.IBufferRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a406d3f69a661d5a3bcb5dded51626c44"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copy</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dst</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferR7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> to <code class="docutils literal notranslate"><span class="pre">dst</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7IBuffer10MemoryType">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager8copyFromERK7IBuffer10MemoryType"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager8copyFromERK7IBuffer10MemoryType"></span><span id="tensorrt_llm::runtime::BufferManager::copyFrom__IBufferCR.MemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a3101d71e18fa84154b687236487dfc5f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyFrom</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7IBuffer10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> into a new <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> with a potentially different memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7ITensor10MemoryType">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager8copyFromERK7ITensor10MemoryType"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager8copyFromERK7ITensor10MemoryType"></span><span id="tensorrt_llm::runtime::BufferManager::copyFrom__ITensorCR.MemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a80a336bdfbd0faf2d76e470354bcddf0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyFrom</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7ITensor10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> into a new <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> with a potentially different memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10IBufferPtrRKNSt6vectorI1TEE10MemoryType">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime13BufferManager8copyFromERKNSt6vectorI1TEE10MemoryType"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime13BufferManager8copyFromERKNSt6vectorI1TEE10MemoryType"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a51f44f1cf97f9c0b686048d37291ea5a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyFrom</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10IBufferPtrRKNSt6vectorI1TEE10MemoryType" title="tensorrt_llm::runtime::BufferManager::copyFrom::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10IBufferPtrRKNSt6vectorI1TEE10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> into a new <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> with a potentially different memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrP1TN8nvinfer14DimsE10MemoryType">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime13BufferManager8copyFromEP1TN8nvinfer14DimsE10MemoryType"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime13BufferManager8copyFromEP1TN8nvinfer14DimsE10MemoryType"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1ac42305510a739c47d3c4358b97560f3e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyFrom</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrP1TN8nvinfer14DimsE10MemoryType" title="tensorrt_llm::runtime::BufferManager::copyFrom::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">src</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrP1TN8nvinfer14DimsE10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> into a new <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> with a potentially different memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrRKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime13BufferManager8copyFromERKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime13BufferManager8copyFromERKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a8feac5727a72ee0d1c50b3ff5ee4e979"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">copyFrom</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrRKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType" title="tensorrt_llm::runtime::BufferManager::copyFrom::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">src</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrRKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType" title="Link to this definition">#</a><br /></dt>
<dd><p>Copy <code class="docutils literal notranslate"><span class="pre">src</span></code> into a new <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> with a potentially different memory type. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager9getStreamEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager9getStreamEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager9getStreamEv"></span><span id="tensorrt_llm::runtime::BufferManager::getStreamC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a895d88127b8bc55f374727908284e352"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getStream</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager9getStreamEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Get the underlying cuda stream. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager18memoryPoolReservedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager18memoryPoolReservedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager18memoryPoolReservedEv"></span><span id="tensorrt_llm::runtime::BufferManager::memoryPoolReservedC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a92a21be0a954deb04c9c6e8540a2141b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">memoryPoolReserved</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager18memoryPoolReservedEv" title="Link to this definition">#</a><br /></dt>
<dd><p>The current size of the memory reserved by the memory pool. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolUsedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager14memoryPoolUsedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager14memoryPoolUsedEv"></span><span id="tensorrt_llm::runtime::BufferManager::memoryPoolUsedC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a73ae2808af584ba85c1a61134f025659"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">memoryPoolUsed</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolUsedEv" title="Link to this definition">#</a><br /></dt>
<dd><p>The current size of the memory used by the memory pool. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolFreeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13BufferManager14memoryPoolFreeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13BufferManager14memoryPoolFreeEv"></span><span id="tensorrt_llm::runtime::BufferManager::memoryPoolFreeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a822c7eaa19d0bdebd460fdc4cf6ee82a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">memoryPoolFree</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolFreeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>The current size of the memory free in the memory pool. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager16memoryPoolTrimToENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager16memoryPoolTrimToENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager16memoryPoolTrimToENSt6size_tE"></span><span id="tensorrt_llm::runtime::BufferManager::memoryPoolTrimTo__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a5e74ccf0e2e2132ae5834c31209f87d3"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">memoryPoolTrimTo</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager16memoryPoolTrimToENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Try to trim the memory reserved by the pool to <code class="docutils literal notranslate"><span class="pre">size</span></code> bytes. This synchronizes implicitly with the stream. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7gpuSyncENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7gpuSyncENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::gpuSync__std::s.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a5906b574a0a12f5842f4e4743fad0d02"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpuSync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size on the GPU, using cudaMalloc. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7gpuSyncEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7gpuSyncEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::gpuSync__nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a0cd267d74d606d1b2f19c64d6dacab9a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gpuSync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions on the GPU, using cudaMalloc. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager3cpuENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager3cpuENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::cpu__std::s.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a7edb3cdde9b85c01fec18f3f4f34282f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cpu</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size on the CPU. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager3cpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager3cpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::cpu__nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a3beb40d9ddabd96774167df148efc676"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cpu</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions on the CPU. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager6pinnedENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager6pinnedENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::pinned__std::s.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a6a58588cfc27c05c1d7dbb2f043672d3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinned</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates a pinned <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size on the CPU. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager6pinnedEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager6pinnedEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::pinned__nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a3cfc4376c5e62876586e1e07e08d3069"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinned</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates a pinned <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions on the CPU. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager10pinnedPoolENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager10pinnedPoolENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::pinnedPool__std::s.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a2e2ddfd2b16255d06ec4490a3bb04301"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinnedPool</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates a pinned <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size on the CPU in the default memory pool. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager10pinnedPoolEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager10pinnedPoolEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::pinnedPool__nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a7a27f3c16ba4d2e85fe825170a7f2ed7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pinnedPool</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates a pinned <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions on the CPU in the default memory pool. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7managedENSt6size_tEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7managedENSt6size_tEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7managedENSt6size_tEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::managed__std::s.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1ace6b7b5bbcd7163c81afdbd7458e1124"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE" title="tensorrt_llm::runtime::BufferManager::IBufferPtr"><span class="n"><span class="pre">IBufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">managed</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7managedENSt6size_tEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> of the given size in UVM. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7managedEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7managedEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7managedEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::managed__nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a101c32638ce8ad5ff06337e322c4bbe9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">managed</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE"><span class="n"><span class="pre">kBYTE_TYPE</span></span></a></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7managedEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions in UVM. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7ipcNvlsENSt3setIiEEN8nvinfer14DimsEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7ipcNvlsENSt3setIiEEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7ipcNvlsENSt3setIiEEN8nvinfer14DimsEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::BufferManager::ipcNvls__std::set:i:.nvinfer1::Dims.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a5b7368b55e92867b8bc815b93ab95056"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE" title="tensorrt_llm::runtime::BufferManager::ITensorPtr"><span class="n"><span class="pre">ITensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipcNvls</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">set</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ranks</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7ipcNvlsENSt3setIiEEN8nvinfer14DimsEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Allocates an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> of the given dimensions for NVLS. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE"></span><span id="tensorrt_llm::runtime::BufferManager::kBYTE_TYPE__auto"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a154196e71e943c8f4c6d0ccacb2dfe3c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kBYTE_TYPE</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kUINT8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager7mStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager7mStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager7mStreamE"></span><span id="tensorrt_llm::runtime::BufferManager::mStream__CudaStreamPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1aeab688938251784e88b66cff01a0dde3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE" title="tensorrt_llm::runtime::BufferManager::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7mStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager5mPoolE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager5mPoolE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager5mPoolE"></span><span id="tensorrt_llm::runtime::BufferManager::mPool__CudaMemPoolPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a1e8ae1268c1ef21cabad78420b798e48"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE" title="tensorrt_llm::runtime::BufferManager::CudaMemPoolPtr"><span class="n"><span class="pre">CudaMemPoolPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPool</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager5mPoolE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13BufferManager9mTrimPoolE">
<span id="_CPPv3N12tensorrt_llm7runtime13BufferManager9mTrimPoolE"></span><span id="_CPPv2N12tensorrt_llm7runtime13BufferManager9mTrimPoolE"></span><span id="tensorrt_llm::runtime::BufferManager::mTrimPool__bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferManager_1a9e8da67a80a77356d5e3c14749a263a0"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTrimPool</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager9mTrimPoolE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
<dl>
<dt class="sig sig-object cpp">
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">::BufferManagerTest</span></dt>
</dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="memorycounters-h">
<h2>memoryCounters.h<a class="headerlink" href="#memorycounters-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCountersE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCountersE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCountersE"></span><span id="tensorrt_llm::runtime::MemoryCounters"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryCounters</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCountersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters10SizeType32E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a7f97eec5349aa1601caef17d277d5f46"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ab5afaef89516ec27d7934229de37e387"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DiffType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ptrdiff_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters14MemoryCountersEv">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters14MemoryCountersEv"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters14MemoryCountersEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::MemoryCounters"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a97911bd1f11691ed4a652bf78e049e08"></span><span class="sig-name descname"><span class="n"><span class="pre">MemoryCounters</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters14MemoryCountersEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getGpuEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters6getGpuEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters6getGpuEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getGpuC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1af804b97c2f575556afca28c10e0fd246"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpu</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getGpuEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getCpuEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters6getCpuEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters6getCpuEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getCpuC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ac559f34eb120bea0f1c499997b6bb7eb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCpu</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getCpuEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters9getPinnedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters9getPinnedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters9getPinnedEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getPinnedC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a4e60cc67231f09228130d20cefb4de0f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPinned</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters9getPinnedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getUVMEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters6getUVMEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters6getUVMEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getUVMC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a1fc4d4828c2838435fbf131d698d035f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getUVM</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getUVMEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedPoolEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters13getPinnedPoolEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters13getPinnedPoolEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getPinnedPoolC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a0937904873a33859769278a94afcb79f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPinnedPool</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedPoolEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getGpuDiffEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters10getGpuDiffEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters10getGpuDiffEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getGpuDiffC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a399a7926f7819096b4f29fa04b3290de"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpuDiff</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getGpuDiffEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getCpuDiffEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters10getCpuDiffEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters10getCpuDiffEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getCpuDiffC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a62f4e6d3f0dd20c4ec6b35b4b6df0d47"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCpuDiff</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getCpuDiffEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedDiffEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters13getPinnedDiffEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters13getPinnedDiffEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getPinnedDiffC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a4571ac85d01e0844b946d57615523ad0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPinnedDiff</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedDiffEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getUVMDiffEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters10getUVMDiffEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters10getUVMDiffEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getUVMDiffC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a29049162d689c1fc63d6df275d13c5d8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getUVMDiff</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getUVMDiffEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters17getPinnedPoolDiffEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters17getPinnedPoolDiffEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters17getPinnedPoolDiffEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getPinnedPoolDiffC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a5a0c1d6212795781ae09a60cfadaa4b6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPinnedPoolDiff</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters17getPinnedPoolDiffEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters8allocateEv10SizeType32">
<span id="_CPPv3I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters8allocateE10SizeType32"></span><span id="_CPPv2I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters8allocateE10SizeType32"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1abbd4ec1ac2e70b15f25f4a11fe28f99c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters8allocateEv10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters8allocateE10MemoryType10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters8allocateE10MemoryType10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters8allocateE10MemoryType10SizeType32"></span><span id="tensorrt_llm::runtime::MemoryCounters::allocate__MemoryType.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a2d7815f9c1dd182a75924b8c3a7262db"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8allocateE10MemoryType10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters10deallocateEv10SizeType32">
<span id="_CPPv3I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters10deallocateE10SizeType32"></span><span id="_CPPv2I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters10deallocateE10SizeType32"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1aea6614ea6ce06f5c28d45bdf296de7f6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deallocate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters10deallocateEv10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters10deallocateE10MemoryType10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters10deallocateE10MemoryType10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters10deallocateE10MemoryType10SizeType32"></span><span id="tensorrt_llm::runtime::MemoryCounters::deallocate__MemoryType.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a874d15b6d6c8f74f0a6ba37ff66d81c4"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">deallocate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memoryType</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10deallocateE10MemoryType10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14MemoryCounters8toStringEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14MemoryCounters8toStringEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14MemoryCounters8toStringEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::toStringC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ad6981e0fc09bbeefdf4b2854fd03d77a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters8toStringEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters11getInstanceEv">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters11getInstanceEv"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters11getInstanceEv"></span><span id="tensorrt_llm::runtime::MemoryCounters::getInstance"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a71d4df0ab707ba942e7a0e5d18f092a5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCountersE" title="tensorrt_llm::runtime::MemoryCounters"><span class="n"><span class="pre">MemoryCounters</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getInstance</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11getInstanceEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE10SizeType32i">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE10SizeType32i"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE10SizeType32i"></span><span id="tensorrt_llm::runtime::MemoryCounters::bytesToString__SizeType32.i"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a31b1552f38254bdd173a8050321a57a5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bytesToString</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bytes</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">precision</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE10SizeType32i" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE8DiffTypei">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE8DiffTypei"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE8DiffTypei"></span><span id="tensorrt_llm::runtime::MemoryCounters::bytesToString__DiffType.i"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ae644945f3b01297e95a0f1724b5b4d8a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bytesToString</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bytes</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">precision</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE8DiffTypei" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mGpuE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters4mGpuE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters4mGpuE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mGpu__std::atomic:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a3e0ea65103a976306619892dc6ebf16a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGpu</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mGpuE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mCpuE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters4mCpuE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters4mCpuE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mCpu__std::atomic:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1aab2d4eec90ab57deb615843586f552f7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCpu</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mCpuE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters7mPinnedE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters7mPinnedE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters7mPinnedE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mPinned__std::atomic:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ad84cb0d4494a355210b3afad7fb5af7a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPinned</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters7mPinnedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mUVME">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters4mUVME"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters4mUVME"></span><span id="tensorrt_llm::runtime::MemoryCounters::mUVM__std::atomic:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a84274b5e6c96b46ad19ef29d80eece02"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUVM</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mUVME" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedPoolE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters11mPinnedPoolE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters11mPinnedPoolE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mPinnedPool__std::atomic:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a170062a3dc6bc26b4c95f526665d93e3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E" title="tensorrt_llm::runtime::MemoryCounters::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPinnedPool</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedPoolE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mGpuDiffE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters8mGpuDiffE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters8mGpuDiffE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mGpuDiff__std::atomic:DiffType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a9f1e18f49dcefe7ad92c21cd7ec3a078"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGpuDiff</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mGpuDiffE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mCpuDiffE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters8mCpuDiffE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters8mCpuDiffE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mCpuDiff__std::atomic:DiffType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a442537225b82fc182cced193b9cc53bc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCpuDiff</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mCpuDiffE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedDiffE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters11mPinnedDiffE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters11mPinnedDiffE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mPinnedDiff__std::atomic:DiffType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a8cfbf01ccdbe867c250798af6b41af81"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPinnedDiff</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedDiffE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mUVMDiffE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters8mUVMDiffE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters8mUVMDiffE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mUVMDiff__std::atomic:DiffType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1ab95b129271bda42cbb4f24545404896a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUVMDiff</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mUVMDiffE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14MemoryCounters15mPinnedPoolDiffE">
<span id="_CPPv3N12tensorrt_llm7runtime14MemoryCounters15mPinnedPoolDiffE"></span><span id="_CPPv2N12tensorrt_llm7runtime14MemoryCounters15mPinnedPoolDiffE"></span><span id="tensorrt_llm::runtime::MemoryCounters::mPinnedPoolDiff__std::atomic:DiffType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MemoryCounters_1a0c7be43e6e4862f11811eae2979634c4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">atomic</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE" title="tensorrt_llm::runtime::MemoryCounters::DiffType"><span class="n"><span class="pre">DiffType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPinnedPoolDiff</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters15mPinnedPoolDiffE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="runtimedefaults-h">
<h2>runtimeDefaults.h<a class="headerlink" href="#runtimedefaults-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE">
<span id="_CPPv3N12tensorrt_llm7runtime15RuntimeDefaultsE"></span><span id="_CPPv2N12tensorrt_llm7runtime15RuntimeDefaultsE"></span><span id="tensorrt_llm::runtime::RuntimeDefaults"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1RuntimeDefaults"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RuntimeDefaults</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::runtime::RuntimeDefaults::RuntimeDefaults__std::optional:std::vector:SizeType32::.std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1RuntimeDefaults_1a4fbd807ecfe1abc3d6747ce3316885d3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RuntimeDefaults</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindowVec</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsEv">
<span id="_CPPv3N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsEv"></span><span id="_CPPv2N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsEv"></span><span id="tensorrt_llm::runtime::RuntimeDefaults::RuntimeDefaults"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1RuntimeDefaults_1ad97400de900b8024bb80c4efc48aae88"></span><span class="sig-name descname"><span class="n"><span class="pre">RuntimeDefaults</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults21maxAttentionWindowVecE">
<span id="_CPPv3N12tensorrt_llm7runtime15RuntimeDefaults21maxAttentionWindowVecE"></span><span id="_CPPv2N12tensorrt_llm7runtime15RuntimeDefaults21maxAttentionWindowVecE"></span><span id="tensorrt_llm::runtime::RuntimeDefaults::maxAttentionWindowVec__std::optional:std::vector:SizeType32::"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1RuntimeDefaults_1ae1ccb7d93441677add1623e581440f40"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxAttentionWindowVec</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults21maxAttentionWindowVecE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15sinkTokenLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime15RuntimeDefaults15sinkTokenLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime15RuntimeDefaults15sinkTokenLengthE"></span><span id="tensorrt_llm::runtime::RuntimeDefaults::sinkTokenLength__std::optional:SizeType32:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1RuntimeDefaults_1aaea0e369a2d34c1b4e64d614281aeec3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">sinkTokenLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15sinkTokenLengthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="ipcutils-h">
<h2>ipcUtils.h<a class="headerlink" href="#ipcutils-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20lamportInitializeAllEPvPvPv6size_t">
<span id="_CPPv3N12tensorrt_llm7runtime20lamportInitializeAllEPvPvPv6size_t"></span><span id="_CPPv2N12tensorrt_llm7runtime20lamportInitializeAllEPvPvPv6size_t"></span><span id="tensorrt_llm::runtime::lamportInitializeAll__voidP.voidP.voidP.s"></span><span class="target" id="ipcUtils_8h_1a40562e2c0ec119fa1918eb42cef0b074"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lamportInitializeAll</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">buffer_0</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">buffer_1</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">buffer_2</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20lamportInitializeAllEPvPvPv6size_t" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13canAccessPeerERK11WorldConfig">
<span id="_CPPv3N12tensorrt_llm7runtime13canAccessPeerERK11WorldConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime13canAccessPeerERK11WorldConfig"></span><span id="tensorrt_llm::runtime::canAccessPeer__WorldConfigCR"></span><span class="target" id="ipcUtils_8h_1ac290a568564018e54160da0a064c4a07"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">canAccessPeer</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13canAccessPeerERK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16AllReduceBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime16AllReduceBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime16AllReduceBuffersE"></span><span id="tensorrt_llm::runtime::AllReduceBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1AllReduceBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AllReduceBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1AllReduceBuffers_1a785db934d1fa831386655fff122f594e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers16AllReduceBuffersE10SizeType3210SizeType3210SizeType3210SizeType32RK13BufferManagerRK11WorldConfigKb">
<span id="_CPPv3N12tensorrt_llm7runtime16AllReduceBuffers16AllReduceBuffersE10SizeType3210SizeType3210SizeType3210SizeType32RK13BufferManagerRK11WorldConfigKb"></span><span id="_CPPv2N12tensorrt_llm7runtime16AllReduceBuffers16AllReduceBuffersE10SizeType3210SizeType3210SizeType3210SizeType32RK13BufferManagerRK11WorldConfigKb"></span><span id="tensorrt_llm::runtime::AllReduceBuffers::AllReduceBuffers__SizeType32.SizeType32.SizeType32.SizeType32.BufferManagerCR.WorldConfigCR.bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1AllReduceBuffers_1ab531d14cdf9443f3730aa2aed8e1635e"></span><span class="sig-name descname"><span class="n"><span class="pre">AllReduceBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">hiddenSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">fakeBuffers</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers16AllReduceBuffersE10SizeType3210SizeType3210SizeType3210SizeType32RK13BufferManagerRK11WorldConfigKb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers18mAllReduceCommPtrsE">
<span id="_CPPv3N12tensorrt_llm7runtime16AllReduceBuffers18mAllReduceCommPtrsE"></span><span id="_CPPv2N12tensorrt_llm7runtime16AllReduceBuffers18mAllReduceCommPtrsE"></span><span id="tensorrt_llm::runtime::AllReduceBuffers::mAllReduceCommPtrs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1AllReduceBuffers_1ab48e63279d11f42d71c3621820d2520c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE" title="tensorrt_llm::runtime::AllReduceBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mAllReduceCommPtrs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers18mAllReduceCommPtrsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers17mIpcMemoryHandlesE">
<span id="_CPPv3N12tensorrt_llm7runtime16AllReduceBuffers17mIpcMemoryHandlesE"></span><span id="_CPPv2N12tensorrt_llm7runtime16AllReduceBuffers17mIpcMemoryHandlesE"></span><span id="tensorrt_llm::runtime::AllReduceBuffers::mIpcMemoryHandles__std::vector:runtime::IpcMemory:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1AllReduceBuffers_1a162c983f7dc981a8c4af57510637e767"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="tensorrt_llm::runtime::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mIpcMemoryHandles</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers17mIpcMemoryHandlesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemoryE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemoryE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemoryE"></span><span id="tensorrt_llm::runtime::IpcMemory"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IpcMemory</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory9BufferPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory9BufferPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory9BufferPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1ab1b59b5e9ca9bae538f4f96f67f54b4c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9BufferPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfigb">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory9IpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfigb"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory9IpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfigb"></span><span id="tensorrt_llm::runtime::IpcMemory::IpcMemory__std::s.BufferManagerCR.WorldConfigCR.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a3375828b692be378adbab4475b734f54"></span><span class="sig-name descname"><span class="n"><span class="pre">IpcMemory</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">bufferSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">openIpc</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfigb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemoryD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemoryD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemoryD0Ev"></span><span id="tensorrt_llm::runtime::IpcMemory::~IpcMemory"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a0f068272bfaeadb2e976b44adf47c484"></span><span class="sig-name descname"><span class="n"><span class="pre">~IpcMemory</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory"></span><span id="tensorrt_llm::runtime::IpcMemory::IpcMemory__IpcMemoryCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1ae3a748873dec82811c4b0014df78d107"></span><span class="sig-name descname"><span class="n"><span class="pre">IpcMemory</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory" title="tensorrt_llm::runtime::IpcMemory::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERK9IpcMemory">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemoryaSERK9IpcMemory"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemoryaSERK9IpcMemory"></span><span id="tensorrt_llm::runtime::IpcMemory::assign-operator__IpcMemoryCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1ad9a357f98e8909799d922dce5f777bd7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="tensorrt_llm::runtime::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="tensorrt_llm::runtime::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERK9IpcMemory" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory"></span><span id="tensorrt_llm::runtime::IpcMemory::IpcMemory__IpcMemoryRR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a611381953a05d2e92e68831e5a459b1a"></span><span class="sig-name descname"><span class="n"><span class="pre">IpcMemory</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory" title="tensorrt_llm::runtime::IpcMemory::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERR9IpcMemory">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemoryaSERR9IpcMemory"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemoryaSERR9IpcMemory"></span><span id="tensorrt_llm::runtime::IpcMemory::assign-operator__IpcMemoryRR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1acd3552f67bf6779ee90c0b2dd471cca4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="tensorrt_llm::runtime::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE" title="tensorrt_llm::runtime::IpcMemory"><span class="n"><span class="pre">IpcMemory</span></span></a><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERR9IpcMemory" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9IpcMemory11getCommPtrsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9IpcMemory11getCommPtrsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9IpcMemory11getCommPtrsEv"></span><span id="tensorrt_llm::runtime::IpcMemory::getCommPtrsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1ad92163a8d4f3b895654e67222be18aac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getCommPtrs</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9IpcMemory11getCommPtrsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory10FLAGS_SIZEE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory10FLAGS_SIZEE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory10FLAGS_SIZEE"></span><span id="tensorrt_llm::runtime::IpcMemory::FLAGS_SIZE__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a3770f9deca8b19095cbe4f0268664265"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FLAGS_SIZE</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kernels</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">MAX_ALL_REDUCE_BLOCKS</span></span><span class="w"> </span><span class="o"><span class="pre">+</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">)</span></span><span class="w"> </span><span class="o"><span class="pre">*</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><span class="n"><span class="pre">uint32_t</span></span><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory10FLAGS_SIZEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory17allocateIpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfig">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory17allocateIpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory17allocateIpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfig"></span><span id="tensorrt_llm::runtime::IpcMemory::allocateIpcMemory__std::s.BufferManagerCR.WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a68894ef110ccba3296dde9e7dff72e61"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allocateIpcMemory</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">bufferSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory17allocateIpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory16destroyIpcMemoryEv">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory16destroyIpcMemoryEv"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory16destroyIpcMemoryEv"></span><span id="tensorrt_llm::runtime::IpcMemory::destroyIpcMemory"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a7b7c035819f96cb37702472ea179c33b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">destroyIpcMemory</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory16destroyIpcMemoryEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory7mTpRankE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory7mTpRankE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory7mTpRankE"></span><span id="tensorrt_llm::runtime::IpcMemory::mTpRank__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a9ce83e03d280dad785bc830c997dad90"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTpRank</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory7mTpRankE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory9mCommPtrsE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory9mCommPtrsE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory9mCommPtrsE"></span><span id="tensorrt_llm::runtime::IpcMemory::mCommPtrs__std::vector:voidP:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1aa15ac983a61b79976435b4d067daccc2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mCommPtrs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9mCommPtrsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory7mBufferE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory7mBufferE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory7mBufferE"></span><span id="tensorrt_llm::runtime::IpcMemory::mBuffer__BufferPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a7907761a17efa256fdbcbe8bb95a26b2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9BufferPtrE" title="tensorrt_llm::runtime::IpcMemory::BufferPtr"><span class="n"><span class="pre">BufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBuffer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory7mBufferE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9IpcMemory8mOpenIpcE">
<span id="_CPPv3N12tensorrt_llm7runtime9IpcMemory8mOpenIpcE"></span><span id="_CPPv2N12tensorrt_llm7runtime9IpcMemory8mOpenIpcE"></span><span id="tensorrt_llm::runtime::IpcMemory::mOpenIpc__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IpcMemory_1a3486fbaa3a609c6e364f5e29557ec630"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOpenIpc</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory8mOpenIpcE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="tllmlogger-h">
<h2>tllmLogger.h<a class="headerlink" href="#tllmlogger-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10TllmLoggerE">
<span id="_CPPv3N12tensorrt_llm7runtime10TllmLoggerE"></span><span id="_CPPv2N12tensorrt_llm7runtime10TllmLoggerE"></span><span id="tensorrt_llm::runtime::TllmLogger"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1TllmLogger"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TllmLogger</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ILogger</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10TllmLoggerE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10TllmLogger3logE8SeverityPKN8nvinfer19AsciiCharE">
<span id="_CPPv3N12tensorrt_llm7runtime10TllmLogger3logE8SeverityPKN8nvinfer19AsciiCharE"></span><span id="_CPPv2N12tensorrt_llm7runtime10TllmLogger3logE8SeverityPKN8nvinfer19AsciiCharE"></span><span id="tensorrt_llm::runtime::TllmLogger::log__Severity.nvinfer1::AsciiCharCP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1TllmLogger_1a052c775ee14bc0d741d26d28c5b3f311"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">log</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">Severity</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">severity</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">AsciiChar</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">msg</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger3logE8SeverityPKN8nvinfer19AsciiCharE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10TllmLogger8getLevelEv">
<span id="_CPPv3N12tensorrt_llm7runtime10TllmLogger8getLevelEv"></span><span id="_CPPv2N12tensorrt_llm7runtime10TllmLogger8getLevelEv"></span><span id="tensorrt_llm::runtime::TllmLogger::getLevel"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1TllmLogger_1ae6b565ac5ee12cfcd305c0f2c0bd4b1e"></span><span class="n"><span class="pre">Severity</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLevel</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger8getLevelEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10TllmLogger8setLevelE8Severity">
<span id="_CPPv3N12tensorrt_llm7runtime10TllmLogger8setLevelE8Severity"></span><span id="_CPPv2N12tensorrt_llm7runtime10TllmLogger8setLevelE8Severity"></span><span id="tensorrt_llm::runtime::TllmLogger::setLevel__Severity"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1TllmLogger_1a02ca7ebe0eec266f8b6ab4b66e9f0275"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLevel</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">Severity</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">level</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger8setLevelE8Severity" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="gptdecoder-h">
<h2>gptDecoder.h<a class="headerlink" href="#gptdecoder-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm6layersE">
<span id="_CPPv3N12tensorrt_llm6layersE"></span><span id="_CPPv2N12tensorrt_llm6layersE"></span><span id="tensorrt_llm::layers"></span><span class="target" id="namespacetensorrt__llm_1_1layers"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">layers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm6layersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20getDefaultBatchSlotsEN7runtime10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime20getDefaultBatchSlotsEN7runtime10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime20getDefaultBatchSlotsEN7runtime10SizeType32E"></span><span id="tensorrt_llm::runtime::getDefaultBatchSlots__runtime::SizeType32"></span><span class="target" id="gptDecoder_8h_1a4be83ec24d8980ca9d74f63e772669e6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDefaultBatchSlots</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20getDefaultBatchSlotsEN7runtime10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd><p>Helper function to produce batch slots [0, 1, …, batchSize - 1] for paths that do not explicitly provide batch slots to the decoder. </p>
</dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime10GptDecoderE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime10GptDecoderE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime10GptDecoderE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GptDecoder</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE" title="tensorrt_llm::runtime::IGptDecoder"><span class="n"><span class="pre">IGptDecoder</span></span></a><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime10GptDecoderE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a8b5c0f02247cc75b5d121681c1990d3b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE" title="tensorrt_llm::runtime::BufferManager::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1ac3e1b92c75c404260a36d3d7e873231b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder10GptDecoderERKN8executor12DecodingModeE6size_t6size_t6size_t6size_t6size_tRK13CudaStreamPtrNSt10shared_ptrIK25SpeculativeDecodingModuleEE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder10GptDecoderERKN8executor12DecodingModeE6size_t6size_t6size_t6size_t6size_tRK13CudaStreamPtrNSt10shared_ptrIK25SpeculativeDecodingModuleEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder10GptDecoderERKN8executor12DecodingModeE6size_t6size_t6size_t6size_t6size_tRK13CudaStreamPtrNSt10shared_ptrIK25SpeculativeDecodingModuleEE"></span><span id="tensorrt_llm::runtime::GptDecoder::GptDecoder__executor::DecodingModeCR.s.s.s.s.s.CudaStreamPtrCR.std::shared_ptr:SpeculativeDecodingModuleC:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a32447c408fdde4dade4948f894f754cb"></span><span class="sig-name descname"><span class="n"><span class="pre">GptDecoder</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSizePadded</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE" title="tensorrt_llm::runtime::GptDecoder::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">speculativeDecodingModule</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder10GptDecoderERKN8executor12DecodingModeE6size_t6size_t6size_t6size_t6size_tRK13CudaStreamPtrNSt10shared_ptrIK25SpeculativeDecodingModuleEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"></span><span id="tensorrt_llm::runtime::GptDecoder::setup__SamplingConfigCR.s.TensorConstPtrCR.std::optional:DecodingOutput:CR.std::optional:std::vector:decoder_batch::Request:C:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a5f4053070f354e9a1a4a55b35553a980"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setup</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">TensorConstPtr</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7RequestE" title="tensorrt_llm::runtime::decoder_batch::Request"><span class="n"><span class="pre">Request</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">requests</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"></span><span id="tensorrt_llm::runtime::GptDecoder::forwardAsync__DecodingOutputR.DecodingInputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1af707cd75a441ad7c639536c75459c65e"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardAsync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"></span><span id="tensorrt_llm::runtime::GptDecoder::forwardSync__DecodingOutputR.DecodingInputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a5418b9d72fb9dc359ecd5e1963ef6dfa"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardSync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder17getSamplingConfigEv">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder17getSamplingConfigEv"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder17getSamplingConfigEv"></span><span id="tensorrt_llm::runtime::GptDecoder::getSamplingConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1af3947dd72df40649ce40d49a5f51c4dd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSamplingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder17getSamplingConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"></span><span id="tensorrt_llm::runtime::GptDecoder::disableLookahead__std::optional:SamplingConfig:CR.SizeType32.TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1aeef976b7acb0e0956b0380f51b8c7044"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookahead</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">TensorConstPtr</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder8mManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder8mManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder8mManagerE"></span><span id="tensorrt_llm::runtime::GptDecoder::mManager__std::shared_ptr:BufferManager:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a0abebbe60a46d72ef92f97e9b00c56e9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder8mManagerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder19mDynamicDecodeLayerE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder19mDynamicDecodeLayerE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder19mDynamicDecodeLayerE"></span><span id="tensorrt_llm::runtime::GptDecoder::mDynamicDecodeLayer__std::shared_ptr:tensorrt_llm::layers::DynamicDecodeLayer:T::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1ae47acd96842e4ebc4f34af4fc4370746"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm6layersE" title="tensorrt_llm::layers"><span class="n"><span class="pre">layers</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DynamicDecodeLayer</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime10GptDecoderE" title="tensorrt_llm::runtime::GptDecoder::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDynamicDecodeLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder19mDynamicDecodeLayerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder23mDecodingLayerWorkspaceE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder23mDecodingLayerWorkspaceE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder23mDecodingLayerWorkspaceE"></span><span id="tensorrt_llm::runtime::GptDecoder::mDecodingLayerWorkspace__std::shared_ptr:tensorrt_llm::runtime::DecodingLayerWorkspace:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a1a7bb40360534e2fa537ee4ca7122d39"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DecodingLayerWorkspace</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodingLayerWorkspace</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder23mDecodingLayerWorkspaceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder15mSamplingConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder15mSamplingConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder15mSamplingConfigE"></span><span id="tensorrt_llm::runtime::GptDecoder::mSamplingConfig__SamplingConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1ac921712531dd8bb357fe6787f036db97"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSamplingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder15mSamplingConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder13mMaxBatchSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder13mMaxBatchSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder13mMaxBatchSizeE"></span><span id="tensorrt_llm::runtime::GptDecoder::mMaxBatchSize__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a44386435d3e8b7eabd481c1df21ae61f"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBatchSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13mMaxBatchSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder10mVocabSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder10mVocabSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder10mVocabSizeE"></span><span id="tensorrt_llm::runtime::GptDecoder::mVocabSize__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a0d89d86d7ac641b9b1d443e6d22051ce"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mVocabSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder10mVocabSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder16mVocabSizePaddedE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder16mVocabSizePaddedE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder16mVocabSizePaddedE"></span><span id="tensorrt_llm::runtime::GptDecoder::mVocabSizePadded__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a962b3a0289469611233768cf98b2baae"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mVocabSizePadded</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder16mVocabSizePaddedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10GptDecoder13mDecodingModeE">
<span id="_CPPv3N12tensorrt_llm7runtime10GptDecoder13mDecodingModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10GptDecoder13mDecodingModeE"></span><span id="tensorrt_llm::runtime::GptDecoder::mDecodingMode__executor::DecodingMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoder_1a30eb6c5c819fe128207960862c893d02"></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecodingMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13mDecodingModeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoderE">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoderE"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoderE"></span><span id="tensorrt_llm::runtime::IGptDecoder"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IGptDecoder</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE" title="Link to this definition">#</a><br /></dt>
<dd><p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1GptDecoder"><span class="std std-ref">tensorrt_llm::runtime::GptDecoder&lt; T &gt;</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a2e73d86e70f2cab27226a5339c34b035"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a54bda2f28e9738845ad2bc84d4d2335b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoderD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoderD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoderD0Ev"></span><span id="tensorrt_llm::runtime::IGptDecoder::~IGptDecoder"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a5ad01b4d1a088b47e56ce6bed969cb57"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~IGptDecoder</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"></span><span id="tensorrt_llm::runtime::IGptDecoder::setup__SamplingConfigCR.s.TensorConstPtrCR.std::optional:DecodingOutput:CR.std::optional:std::vector:decoder_batch::Request:C:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1af15d3c58f50e58cac3f44d4580e6db84"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setup</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE" title="tensorrt_llm::runtime::IGptDecoder::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7RequestE" title="tensorrt_llm::runtime::decoder_batch::Request"><span class="n"><span class="pre">Request</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">requests</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"></span><span id="tensorrt_llm::runtime::IGptDecoder::forwardAsync__DecodingOutputR.DecodingInputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a8f87707fea456a6decd013b6b831d336"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardAsync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"></span><span id="tensorrt_llm::runtime::IGptDecoder::forwardSync__DecodingOutputR.DecodingInputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1ae06881ec486e5aadec7d8df477e214e4"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardSync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="tensorrt_llm::runtime::DecodingOutput"><span class="n"><span class="pre">DecodingOutput</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="tensorrt_llm::runtime::DecodingInput"><span class="n"><span class="pre">DecodingInput</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder17getSamplingConfigEv">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder17getSamplingConfigEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder17getSamplingConfigEv"></span><span id="tensorrt_llm::runtime::IGptDecoder::getSamplingConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1aae2ec40e54ccd288a7c548f09b4f3eb1"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSamplingConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder17getSamplingConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"></span><span id="tensorrt_llm::runtime::IGptDecoder::disableLookahead__std::optional:SamplingConfig:CR.SizeType32.TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a732a15be45afd1f693396e7c88c629af"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookahead</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE" title="tensorrt_llm::runtime::IGptDecoder::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11IGptDecoder6createERKN8executor12DecodingModeEN8nvinfer18DataTypeE6size_t6size_t6size_t6size_t6size_tRKN13BufferManager13CudaStreamPtrERKNSt10shared_ptrIK25SpeculativeDecodingModuleEE">
<span id="_CPPv3N12tensorrt_llm7runtime11IGptDecoder6createERKN8executor12DecodingModeEN8nvinfer18DataTypeE6size_t6size_t6size_t6size_t6size_tRKN13BufferManager13CudaStreamPtrERKNSt10shared_ptrIK25SpeculativeDecodingModuleEE"></span><span id="_CPPv2N12tensorrt_llm7runtime11IGptDecoder6createERKN8executor12DecodingModeEN8nvinfer18DataTypeE6size_t6size_t6size_t6size_t6size_tRKN13BufferManager13CudaStreamPtrERKNSt10shared_ptrIK25SpeculativeDecodingModuleEE"></span><span id="tensorrt_llm::runtime::IGptDecoder::create__executor::DecodingModeCR.nvinfer1::DataType.s.s.s.s.s.BufferManager::CudaStreamPtrCR.std::shared_ptr:SpeculativeDecodingModuleC:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoder_1a16c5211cd06c17f8100edacde78c6477"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE" title="tensorrt_llm::runtime::IGptDecoder"><span class="n"><span class="pre">IGptDecoder</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">create</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSizePadded</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE" title="tensorrt_llm::runtime::BufferManager::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">speculativeDecodingModule</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder6createERKN8executor12DecodingModeEN8nvinfer18DataTypeE6size_t6size_t6size_t6size_t6size_tRKN13BufferManager13CudaStreamPtrERKNSt10shared_ptrIK25SpeculativeDecodingModuleEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="cudaevent-h">
<h2>cudaEvent.h<a class="headerlink" href="#cudaevent-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEventE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEventE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEventE"></span><span id="tensorrt_llm::runtime::CudaEvent"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaEvent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent7pointerE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent7pointerE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a765323cdb24844d31c8f9e354a5194f8"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pointer</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">cudaEvent_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventEj">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent9CudaEventEj"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent9CudaEventEj"></span><span id="tensorrt_llm::runtime::CudaEvent::CudaEvent__unsigned-i"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1aae9d65fd3cc3d42763c1219710dcd6ac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaEvent</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">unsigned</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">flags</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">cudaEventDisableTiming</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventEj" title="Link to this definition">#</a><br /></dt>
<dd><p>Creates a new cuda event. The event will be destroyed in the destructor.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>flags</strong> – Flags for event creation. By default, event timing is disabled. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventE7pointerb">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent9CudaEventE7pointerb"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent9CudaEventE7pointerb"></span><span id="tensorrt_llm::runtime::CudaEvent::CudaEvent__pointer.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a8ac4cb3f5ac924e72862c1c5fd033cbd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaEvent</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">event</span></span></em>, <em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ownsEvent</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventE7pointerb" title="Link to this definition">#</a><br /></dt>
<dd><p>Pass an existing cuda event to this object.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>event</strong> – The event to pass to this object. </p></li>
<li><p><strong>ownsEvent</strong> – Whether this object owns the event and destroys it in the destructor. </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9CudaEvent3getEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9CudaEvent3getEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9CudaEvent3getEv"></span><span id="tensorrt_llm::runtime::CudaEvent::getC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a530c756fe9e6ad149b813659b3644f16"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">get</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent3getEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the event associated with this object. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9CudaEvent11synchronizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9CudaEvent11synchronizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9CudaEvent11synchronizeEv"></span><span id="tensorrt_llm::runtime::CudaEvent::synchronizeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a9216cdfafea99849fb47b1be60fb1a7e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">synchronize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent11synchronizeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Synchronizes the event. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent12element_typeE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent12element_typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent12element_typeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a411326cde2f9f947c5d92abe724ebb34"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">element_type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_pointer_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent12element_typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent8EventPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent8EventPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent8EventPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1ac5468fc310bab51276ed6d6212e7240f"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EventPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent12element_typeE" title="tensorrt_llm::runtime::CudaEvent::element_type"><span class="n"><span class="pre">element_type</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7DeleterE" title="tensorrt_llm::runtime::CudaEvent::Deleter"><span class="n"><span class="pre">Deleter</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent8EventPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent6mEventE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent6mEventE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent6mEventE"></span><span id="tensorrt_llm::runtime::CudaEvent::mEvent__EventPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1a3231de6317de1857e5f699a94d59dcf2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent8EventPtrE" title="tensorrt_llm::runtime::CudaEvent::EventPtr"><span class="n"><span class="pre">EventPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEvent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent6mEventE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent7DeleterE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent7DeleterE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent7DeleterE"></span><span id="tensorrt_llm::runtime::CudaEvent::Deleter"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1_1Deleter"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7DeleterE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEb">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEb"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEb"></span><span id="tensorrt_llm::runtime::CudaEvent::Deleter::Deleter__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1_1Deleter_1a7807c6a0240c9d01f09dbde19b422d05"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ownsEvent</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEv">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEv"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEv"></span><span id="tensorrt_llm::runtime::CudaEvent::Deleter::Deleter"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1_1Deleter_1aec3a7bdb695dc6598d6c6b8c5cbd4598"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9CudaEvent7DeleterclE7pointer">
<span id="_CPPv3NK12tensorrt_llm7runtime9CudaEvent7DeleterclE7pointer"></span><span id="_CPPv2NK12tensorrt_llm7runtime9CudaEvent7DeleterclE7pointer"></span><span id="tensorrt_llm::runtime::CudaEvent::Deleter::call-operator__pointerCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1_1Deleter_1a82af09350919fceb1f0f4c43d1822d37"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">()</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent7DeleterclE7pointer" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter10mOwnsEventE">
<span id="_CPPv3N12tensorrt_llm7runtime9CudaEvent7Deleter10mOwnsEventE"></span><span id="_CPPv2N12tensorrt_llm7runtime9CudaEvent7Deleter10mOwnsEventE"></span><span id="tensorrt_llm::runtime::CudaEvent::Deleter::mOwnsEvent__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaEvent_1_1Deleter_1a4df7640a97930eaf43a1ca952f3f919f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOwnsEvent</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter10mOwnsEventE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="modelconfig-h">
<h2>modelConfig.h<a class="headerlink" href="#modelconfig-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfigE"></span><span id="tensorrt_llm::runtime::ModelConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariantE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariantE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModelVariant</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGptE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGptE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGptE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7aeb854c1219d60212d16ad06f0d454dc8"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGpt</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGptE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant8kChatGlmE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant8kChatGlmE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant8kChatGlmE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7a3f33cf3b2415ad1e408896306d35d786"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kChatGlm</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant8kChatGlmE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGlmE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGlmE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGlmE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7a30a41043cc5346b26f02b1a3e1cf54d5"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGlm</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGlmE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant6kMambaE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant6kMambaE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant6kMambaE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7a434d259e93688631e9f679f6af07a30f"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMamba</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant6kMambaE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant15kRecurrentGemmaE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant15kRecurrentGemmaE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant15kRecurrentGemmaE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7a45c696d9e1b5effa650b69ac887a9983"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kRecurrentGemma</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant15kRecurrentGemmaE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant7kEncDecE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12ModelVariant7kEncDecE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12ModelVariant7kEncDecE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a46a3930b326403036a2d826741aeacc7adafb6f4a6d945f658e4da3f2a8dd4fd7"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEncDec</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant7kEncDecE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9LayerTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9LayerTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad27648d68c7e473f9189b7843f3a8c00"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LayerType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kATTENTIONE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9LayerType10kATTENTIONE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9LayerType10kATTENTIONE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad27648d68c7e473f9189b7843f3a8c00a7c43472f56b13f7582947b4ef8e73f98"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTENTION</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kATTENTIONE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kRECURRENTE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9LayerType10kRECURRENTE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9LayerType10kRECURRENTE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad27648d68c7e473f9189b7843f3a8c00a82be6fea77422f4810f7dc2cbe14bc2e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kRECURRENT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kRECURRENTE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType7kLINEARE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9LayerType7kLINEARE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9LayerType7kLINEARE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad27648d68c7e473f9189b7843f3a8c00afd0bda8f85b35011bdcde415691fc36f"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLINEAR</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType7kLINEARE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType5kNOOPE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9LayerType5kNOOPE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9LayerType5kNOOPE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad27648d68c7e473f9189b7843f3a8c00ac1ccb8aad06b583f8b824ce7c4446132"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kNOOP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType5kNOOPE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a48d8743f0628ea2034042c6bcfed8517"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a48d8743f0628ea2034042c6bcfed8517ad2fadda76e2fe15f09844dafcb790d7e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCONTINUOUS</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType6kPAGEDE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11KVCacheType6kPAGEDE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11KVCacheType6kPAGEDE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a48d8743f0628ea2034042c6bcfed8517aad012df8c2260ef3b7e51d63f533dc16"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kPAGED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType6kPAGEDE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType9kDISABLEDE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11KVCacheType9kDISABLEDE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11KVCacheType9kDISABLEDE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a48d8743f0628ea2034042c6bcfed8517a02bd32be241163867c72873cf95845a1"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDISABLED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType9kDISABLEDE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aee73725fb2f9e639356718d384025363"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ManageWeightsType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType9kDisabledE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType9kDisabledE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType9kDisabledE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aee73725fb2f9e639356718d384025363a7b04d8795f5fd03c761ce70dab985fee"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDisabled</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType9kDisabledE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType8kEnabledE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType8kEnabledE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType8kEnabledE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aee73725fb2f9e639356718d384025363ade1ec1870756a80dffbdc4cda53153e0"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEnabled</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType8kEnabledE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11ModelConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11ModelConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11ModelConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::ModelConfig__SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a0beb690f207ae612e46df0f448a01225"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModelConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbLayers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbAttentionLayers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbRnnLayers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbHeads</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">hiddenSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11ModelConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getVocabSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12getVocabSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12getVocabSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getVocabSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad2c7526a1c003a8a4278549db295aed0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getVocabSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getVocabSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getVocabSizePaddedE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18getVocabSizePaddedE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18getVocabSizePaddedE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getVocabSizePadded__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a44b691295aae063983d55b0a4cf32299"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getVocabSizePadded</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">worldSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getVocabSizePaddedE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig16countLocalLayersE9LayerType10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig16countLocalLayersE9LayerType10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig16countLocalLayersE9LayerType10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::countLocalLayers__LayerType.SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad422a96486029b9b47f1e346cd7d2de3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">countLocalLayers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="tensorrt_llm::runtime::ModelConfig::LayerType"><span class="n"><span class="pre">LayerType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">layerType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16countLocalLayersE9LayerType10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getFirstLocalLayerE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18getFirstLocalLayerE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18getFirstLocalLayerE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getFirstLocalLayer__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2c819f0d4717a6ad56c2f701f0ff1698"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getFirstLocalLayer</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getFirstLocalLayerE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20countLowerRankLayersE9LayerType10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20countLowerRankLayersE9LayerType10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20countLowerRankLayersE9LayerType10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::countLowerRankLayers__LayerType.SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4c9cabd1675a0db58bce743a0ac0470e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">countLowerRankLayers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="tensorrt_llm::runtime::ModelConfig::LayerType"><span class="n"><span class="pre">LayerType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">layerType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20countLowerRankLayersE9LayerType10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getNbLayersE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig11getNbLayersE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig11getNbLayersE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getNbLayers__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aba756e17c1d83a61adf10f12a3787479"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNbLayers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getNbLayersE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getNbAttentionLayersE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20getNbAttentionLayersE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20getNbAttentionLayersE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getNbAttentionLayers__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae7312b4e1c8f27bcba6305fa0c4eced5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNbAttentionLayers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getNbAttentionLayersE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getNbRnnLayersE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getNbRnnLayersE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getNbRnnLayersE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getNbRnnLayers__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac825a4cdeeffed1ea53bec8d5fa6bba6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNbRnnLayers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getNbRnnLayersE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig10getNbHeadsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig10getNbHeadsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig10getNbHeadsEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getNbHeadsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac4e85f0d390d2d281cbacc1e6d2c2a7d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNbHeads</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig10getNbHeadsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getNbKvHeadsE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12getNbKvHeadsE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12getNbKvHeadsE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::getNbKvHeads__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa5d5e18d61f0198c9c2ee02115841e81"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNbKvHeads</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">layerIdx</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getNbKvHeadsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12setNbKvHeadsE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12setNbKvHeadsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12setNbKvHeadsE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setNbKvHeads__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a70bc4cfb6e0ef27c086735587a275072"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNbKvHeads</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbKvHeads</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setNbKvHeadsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17setNbCrossKvHeadsE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17setNbCrossKvHeadsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17setNbCrossKvHeadsE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setNbCrossKvHeads__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a89e6744eaba2b4fae56a6310dd664d2f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNbCrossKvHeads</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nbKvHeads</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setNbCrossKvHeadsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getHiddenSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig13getHiddenSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig13getHiddenSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getHiddenSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a95aabae940f79963d6a59216cc3e8c62"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getHiddenSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getHiddenSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getEncoderHiddenSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20getEncoderHiddenSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20getEncoderHiddenSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getEncoderHiddenSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af5ff1b00564efeee5358302a34ecbb6e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getEncoderHiddenSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getEncoderHiddenSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20setEncoderHiddenSizeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20setEncoderHiddenSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20setEncoderHiddenSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setEncoderHiddenSize__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae95ad1e5d827d9f266ab2c360eb91b42"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEncoderHiddenSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">encoderHiddenSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setEncoderHiddenSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getSizePerHeadEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getSizePerHeadEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getSizePerHeadEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getSizePerHeadCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1adc06556fe8bcc02ebd448a091b758cb7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSizePerHead</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getSizePerHeadEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setSizePerHeadE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setSizePerHeadE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setSizePerHeadE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setSizePerHead__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4d903cbc2da841a66e0c8f3c77bf19d7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSizePerHead</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sizePerHead</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setSizePerHeadE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getDataTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig11getDataTypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getDataTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac5582fd24365d583731bbd8fc62dbe5a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useGptAttentionPluginCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa4708970b05b37949e471cb815cf32b4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useGptAttentionPlugin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useGemmAllReducePluginCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab1bb80e0fcde35159c4d31af997f2a21"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useGemmAllReducePlugin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEb"></span><span id="tensorrt_llm::runtime::ModelConfig::useGptAttentionPlugin__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af550b4ebc4dae83ea6951620d40d887d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useGptAttentionPlugin</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useGptAttentionPlugin</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEb"></span><span id="tensorrt_llm::runtime::ModelConfig::useGemmAllReducePlugin__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a496cf82d9fc9fa422ae23d8c200d443f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useGemmAllReducePlugin</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useGemmAllReducePlugin</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useMambaConv1dPluginCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1afcd536ea3183f61555d4a9bfb0a4baab"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMambaConv1dPlugin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEb"></span><span id="tensorrt_llm::runtime::ModelConfig::useMambaConv1dPlugin__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a6b689c7681f63d3e963680cc9f95917d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMambaConv1dPlugin</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useMambaConv1dPlugin</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14usePackedInputEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14usePackedInputEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14usePackedInputEv"></span><span id="tensorrt_llm::runtime::ModelConfig::usePackedInputCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1adafa102c76fd0db86c206f5969366038"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePackedInput</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14usePackedInputEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14usePackedInputEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14usePackedInputEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14usePackedInputEb"></span><span id="tensorrt_llm::runtime::ModelConfig::usePackedInput__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4784e5f1368098b1f78e81e4199c4985"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePackedInput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inputPacked</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14usePackedInputEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig13usePagedStateEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig13usePagedStateEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig13usePagedStateEv"></span><span id="tensorrt_llm::runtime::ModelConfig::usePagedStateCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a08e0e846ca8abd00f9e929c916fcd4c0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePagedState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13usePagedStateEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13usePagedStateEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13usePagedStateEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13usePagedStateEb"></span><span id="tensorrt_llm::runtime::ModelConfig::usePagedState__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af20937bb5b5da5827b48174c510d15de"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePagedState</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pagedState</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13usePagedStateEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getTokensPerBlockEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig17getTokensPerBlockEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig17getTokensPerBlockEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getTokensPerBlockCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5fc2b306388f583fbeb355a413eb5a86"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTokensPerBlock</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getTokensPerBlockEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17setTokensPerBlockE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17setTokensPerBlockE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17setTokensPerBlockE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setTokensPerBlock__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5386c582426ad3ce798605a96d889d8c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTokensPerBlock</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">TokensPerBlock</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setTokensPerBlockE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getQuantModeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12getQuantModeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12getQuantModeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getQuantModeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa2ef652e4533b31ce88f4c0d1ad97121"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">QuantMode</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getQuantMode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getQuantModeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12setQuantModeEN6common9QuantModeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12setQuantModeEN6common9QuantModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12setQuantModeEN6common9QuantModeE"></span><span id="tensorrt_llm::runtime::ModelConfig::setQuantMode__common::QuantModeCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae5191acbb52d822a7dccc730af71b387"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setQuantMode</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">QuantMode</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">QuantMode</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setQuantModeEN6common9QuantModeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig24supportsInflightBatchingEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig24supportsInflightBatchingEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig24supportsInflightBatchingEv"></span><span id="tensorrt_llm::runtime::ModelConfig::supportsInflightBatchingCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a39fb96cf15bb5175fcb35ac6158f2e06"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">supportsInflightBatching</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig24supportsInflightBatchingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBatchSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15getMaxBatchSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15getMaxBatchSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxBatchSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1acae205bb08586324fa0ea575f349e91e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBatchSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBatchSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBatchSizeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15setMaxBatchSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15setMaxBatchSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxBatchSize__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1afcbe41c09cd3d1536350cc37ecc0d908"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxBatchSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBatchSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBeamWidthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15getMaxBeamWidthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15getMaxBeamWidthEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxBeamWidthCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae4eb638abf8f95b30b182b134498445c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBeamWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBeamWidthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBeamWidthE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15setMaxBeamWidthE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15setMaxBeamWidthE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxBeamWidth__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a87e57088a3f8e6629d7ace77b02376d4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxBeamWidth</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBeamWidthE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxInputLenEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getMaxInputLenEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getMaxInputLenEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxInputLenCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad54640d8c03c12f153eaf02b9043463c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxInputLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxInputLenEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxInputLenE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setMaxInputLenE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setMaxInputLenE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxInputLen__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af71d7e5e037ec2ebfe5730ed6d26feee"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxInputLen</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxInputLen</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxInputLenE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getMaxSequenceLenEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig17getMaxSequenceLenEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig17getMaxSequenceLenEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxSequenceLenCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8e41be40fd748e091ceb34739c21467d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxSequenceLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getMaxSequenceLenEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17setMaxSequenceLenE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17setMaxSequenceLenE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17setMaxSequenceLenE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxSequenceLen__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1acf44bbab9357ab2591ffa9f218607f19"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxSequenceLen</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLen</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setMaxSequenceLenE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxNumTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15getMaxNumTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15getMaxNumTokensEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxNumTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a837af9ec3c2c57de3fae4939029d174b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxNumTokens</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxNumTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxNumTokensENSt8optionalI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15setMaxNumTokensENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15setMaxNumTokensENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxNumTokens__std::optional:SizeType32:CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af0889ca073da922234c4edad82e71cf9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxNumTokens</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumTokens</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxNumTokensENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMaxEncoderLenEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig16getMaxEncoderLenEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig16getMaxEncoderLenEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxEncoderLenCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3c8ddbde368dce4f139b4df64da2ad06"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxEncoderLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMaxEncoderLenEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMaxEncoderLenE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig16setMaxEncoderLenE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig16setMaxEncoderLenE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxEncoderLen__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a38169800b959be50c524bec9600b07c0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxEncoderLen</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxEncoderLen</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMaxEncoderLenE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15usePromptTuningEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15usePromptTuningEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15usePromptTuningEv"></span><span id="tensorrt_llm::runtime::ModelConfig::usePromptTuningCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a0020d543fa5a3806258f783a988ee1fc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePromptTuning</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15usePromptTuningEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig8useMropeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig8useMropeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig8useMropeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useMropeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a40cc96908df97d5a254e0c5189561ee4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useMrope</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig8useMropeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11setUseMropeEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11setUseMropeEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11setUseMropeEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setUseMrope__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab96860034c07d8387d3496443612e5ad"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseMrope</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useMrope</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11setUseMropeEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig24getMaxPositionEmbeddingsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig24getMaxPositionEmbeddingsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig24getMaxPositionEmbeddingsEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxPositionEmbeddingsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a977d19cd0ff55979f51a6d61a53c8294"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPositionEmbeddings</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig24getMaxPositionEmbeddingsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig24setMaxPositionEmbeddingsE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig24setMaxPositionEmbeddingsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig24setMaxPositionEmbeddingsE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxPositionEmbeddings__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3463fe9d15b9402f45d9a0477ef5ae35"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxPositionEmbeddings</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxPositionEmbeddings</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24setMaxPositionEmbeddingsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getRotaryEmbeddingDimEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig21getRotaryEmbeddingDimEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig21getRotaryEmbeddingDimEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getRotaryEmbeddingDimCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a1b5f10246eed6fd56cfb04bfe8320731"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRotaryEmbeddingDim</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getRotaryEmbeddingDimEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21setRotaryEmbeddingDimE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21setRotaryEmbeddingDimE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21setRotaryEmbeddingDimE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setRotaryEmbeddingDim__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab41aa3d6b8cfecaeaeed2f4598238b49"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRotaryEmbeddingDim</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">rotaryEmbeddingDim</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setRotaryEmbeddingDimE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig30getMaxPromptEmbeddingTableSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig30getMaxPromptEmbeddingTableSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig30getMaxPromptEmbeddingTableSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxPromptEmbeddingTableSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab18461ef064f3e15265b2d823912e60c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPromptEmbeddingTableSize</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig30getMaxPromptEmbeddingTableSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig30setMaxPromptEmbeddingTableSizeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig30setMaxPromptEmbeddingTableSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig30setMaxPromptEmbeddingTableSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxPromptEmbeddingTableSize__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af2fa5f70a41247541ac41becb2ccc397"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxPromptEmbeddingTableSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxPromptEmbeddingTableSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig30setMaxPromptEmbeddingTableSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEv"></span><span id="tensorrt_llm::runtime::ModelConfig::computeContextLogitsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a6ff1b92b7c4079b8c27edc5916ca489c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">computeContextLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEb"></span><span id="tensorrt_llm::runtime::ModelConfig::computeContextLogits__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a76db846b5aa7f84e2eeb99ca291de6cd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">computeContextLogits</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">computeContextLogits</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEv"></span><span id="tensorrt_llm::runtime::ModelConfig::computeGenerationLogitsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af3bad84ea1f489475d210018a5226937"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">computeGenerationLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEb"></span><span id="tensorrt_llm::runtime::ModelConfig::computeGenerationLogits__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a81140bd79cbf1020af663df0c64f32eb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">computeGenerationLogits</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">computeGenerationLogits</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getModelVariantEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15getModelVariantEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15getModelVariantEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getModelVariantC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa1c5395236e029587c0320bcdfc372da"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE" title="tensorrt_llm::runtime::ModelConfig::ModelVariant"><span class="n"><span class="pre">ModelVariant</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getModelVariant</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getModelVariantEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15setModelVariantE12ModelVariant">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15setModelVariantE12ModelVariant"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15setModelVariantE12ModelVariant"></span><span id="tensorrt_llm::runtime::ModelConfig::setModelVariant__ModelVariant"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a6b497bc99ecd2f7a41e6ccb04ee88741"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setModelVariant</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE" title="tensorrt_llm::runtime::ModelConfig::ModelVariant"><span class="n"><span class="pre">ModelVariant</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelVariant</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setModelVariantE12ModelVariant" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig25getMaxDecodingDraftTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig25getMaxDecodingDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig25getMaxDecodingDraftTokensEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxDecodingDraftTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5fea542584ac51d190be4ad18ebb3489"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig25getMaxDecodingDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getMaxDecodingTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20getMaxDecodingTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20getMaxDecodingTokensEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxDecodingTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3c891a1c7a4a982505fc6c6b8df196fd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getMaxDecodingTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setContextFMHAEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setContextFMHAEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setContextFMHAEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setContextFMHA__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a462afcfe43deb94f020e90849000455e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setContextFMHA</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">contextFMHA</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setContextFMHAEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getContextFMHAEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getContextFMHAEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getContextFMHAEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getContextFMHACCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a0f0cfb8b8352a61bcb47e6824f8a6103"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextFMHA</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getContextFMHAEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig19setPagedContextFMHAEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig19setPagedContextFMHAEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig19setPagedContextFMHAEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setPagedContextFMHA__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a002731c2125d9f304d4d61ba481c1ce4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPagedContextFMHA</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pagedContextFMHA</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19setPagedContextFMHAEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig19getPagedContextFMHAEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig19getPagedContextFMHAEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig19getPagedContextFMHAEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getPagedContextFMHACCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae54403bdb030b67cafb6f550bed19337"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPagedContextFMHA</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19getPagedContextFMHAEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18setPpReduceScatterEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18setPpReduceScatterEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18setPpReduceScatterEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setPpReduceScatter__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab164555c0a5261405e87e924c59ec140"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPpReduceScatter</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ppReduceScatter</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18setPpReduceScatterEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getPpReduceScatterEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18getPpReduceScatterEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18getPpReduceScatterEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getPpReduceScatterCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a08abc6e0cd4d9322389a9d8332ae7ea3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPpReduceScatter</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getPpReduceScatterEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig13useLoraPluginEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig13useLoraPluginEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig13useLoraPluginEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useLoraPluginCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac012addc4a1a6bbdc5c253bcd313c75a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useLoraPlugin</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13useLoraPluginEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13useLoraPluginEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13useLoraPluginEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13useLoraPluginEb"></span><span id="tensorrt_llm::runtime::ModelConfig::useLoraPlugin__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1addfdf96d94cb78c38de656530bac978e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useLoraPlugin</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useLoraPlugin</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13useLoraPluginEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLoraModulesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getLoraModulesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getLoraModulesEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getLoraModulesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a394bc28cc8e8f3a925a815dcec0c8400"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getLoraModules</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLoraModulesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLoraModulesERKNSt6vectorI10LoraModuleEE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setLoraModulesERKNSt6vectorI10LoraModuleEE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setLoraModulesERKNSt6vectorI10LoraModuleEE"></span><span id="tensorrt_llm::runtime::ModelConfig::setLoraModules__std::vector:LoraModule:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af3d088c4f0e84585ed054fa593fd4606"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLoraModules</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">loraModules</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLoraModulesERKNSt6vectorI10LoraModuleEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMlpHiddenSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig16getMlpHiddenSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig16getMlpHiddenSizeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMlpHiddenSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a480c6e75519cd425c5fc14d7f0779139"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMlpHiddenSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMlpHiddenSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMlpHiddenSizeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig16setMlpHiddenSizeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig16setMlpHiddenSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMlpHiddenSize__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a45a082bcdab56ba7ffaa4179e2e8aac5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMlpHiddenSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mlpHiddenSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMlpHiddenSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig16isKVCacheEnabledEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig16isKVCacheEnabledEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig16isKVCacheEnabledEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isKVCacheEnabledCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a1c2feb8048a22aa65fa90d2fbcadd192"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isKVCacheEnabled</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16isKVCacheEnabledEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14isPagedKVCacheEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14isPagedKVCacheEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14isPagedKVCacheEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isPagedKVCacheCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a90b9ef0bd3e8cba931e27df219e945b7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isPagedKVCache</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14isPagedKVCacheEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig19isContinuousKVCacheEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig19isContinuousKVCacheEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig19isContinuousKVCacheEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isContinuousKVCacheCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a66964e35ccf3a678d6873fc1d67d8cfe"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isContinuousKVCache</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19isContinuousKVCacheEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getKVCacheTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getKVCacheTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getKVCacheTypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getKVCacheTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a04585fd0af4a92429771ef5ebcc831d1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType"><span class="n"><span class="pre">KVCacheType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKVCacheType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getKVCacheTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setKVCacheTypeE11KVCacheType">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setKVCacheTypeE11KVCacheType"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setKVCacheTypeE11KVCacheType"></span><span id="tensorrt_llm::runtime::ModelConfig::setKVCacheType__KVCacheTypeCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a66418c72ac9b39d9e4903c068ce3e8d9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setKVCacheType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType"><span class="n"><span class="pre">KVCacheType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">kvCacheType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setKVCacheTypeE11KVCacheType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useCrossAttentionEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig17useCrossAttentionEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig17useCrossAttentionEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useCrossAttentionCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aeb95033e26e2ecabad09be0563aa07a5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useCrossAttention</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useCrossAttentionEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseCrossAttentionEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20setUseCrossAttentionEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20setUseCrossAttentionEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setUseCrossAttention__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a1faff3e37890eaffb65df1d29bcf15a3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseCrossAttention</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useCrossAttention</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseCrossAttentionEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20usePositionEmbeddingEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20usePositionEmbeddingEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20usePositionEmbeddingEv"></span><span id="tensorrt_llm::runtime::ModelConfig::usePositionEmbeddingCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1adf5df806638e5486e166ce3036070060"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">usePositionEmbedding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20usePositionEmbeddingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig23setUsePositionEmbeddingEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig23setUsePositionEmbeddingEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig23setUsePositionEmbeddingEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setUsePositionEmbedding__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a992b317fd23cc91af2cb8f99818fec0c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUsePositionEmbedding</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">usePositionEmbedding</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23setUsePositionEmbeddingEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useTokenTypeEmbeddingEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig21useTokenTypeEmbeddingEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig21useTokenTypeEmbeddingEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useTokenTypeEmbeddingCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac8e1ec78f0746e0124ee05ec025d0509"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useTokenTypeEmbedding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useTokenTypeEmbeddingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig24setUseTokenTypeEmbeddingEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig24setUseTokenTypeEmbeddingEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig24setUseTokenTypeEmbeddingEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setUseTokenTypeEmbedding__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a0242017a7f1a4c104e379dd1ec28213f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseTokenTypeEmbedding</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useTokenTypeEmbedding</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24setUseTokenTypeEmbeddingEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxLoraRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getMaxLoraRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getMaxLoraRankEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getMaxLoraRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a34ebc12fa624b6964f4aaf10ab431934"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxLoraRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxLoraRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxLoraRankE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setMaxLoraRankE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setMaxLoraRankE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::setMaxLoraRank__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4c2d224c8626aa07e83e9ea76dbecb7c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxLoraRank</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxLoraRank</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxLoraRankE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig26setSpeculativeDecodingModeE23SpeculativeDecodingMode">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig26setSpeculativeDecodingModeE23SpeculativeDecodingMode"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig26setSpeculativeDecodingModeE23SpeculativeDecodingMode"></span><span id="tensorrt_llm::runtime::ModelConfig::setSpeculativeDecodingMode__SpeculativeDecodingMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad7ec4cac1b5b05796701db1c18eef1c9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSpeculativeDecodingMode</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26setSpeculativeDecodingModeE23SpeculativeDecodingMode" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig28hasSpeculativeDecodingModuleEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig28hasSpeculativeDecodingModuleEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig28hasSpeculativeDecodingModuleEv"></span><span id="tensorrt_llm::runtime::ModelConfig::hasSpeculativeDecodingModuleC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a94f38f7c3078e8b5b8b965681356175c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">hasSpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig28hasSpeculativeDecodingModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig28getSpeculativeDecodingModuleEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig28getSpeculativeDecodingModuleEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig28getSpeculativeDecodingModuleEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getSpeculativeDecodingModuleC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1afc7bbd0919a464d837a489771b06162f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getSpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig28getSpeculativeDecodingModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getSpeculativeDecodingModulePtrC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4a3cc906d7d9cca119da52b3c9609e3f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpeculativeDecodingModulePtr</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getSpeculativeDecodingModulePtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a32cdeed2f3b2f53505371743a33f0c2d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpeculativeDecodingModulePtr</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig28setSpeculativeDecodingModuleERKNSt10shared_ptrI25SpeculativeDecodingModuleEE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig28setSpeculativeDecodingModuleERKNSt10shared_ptrI25SpeculativeDecodingModuleEE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig28setSpeculativeDecodingModuleERKNSt10shared_ptrI25SpeculativeDecodingModuleEE"></span><span id="tensorrt_llm::runtime::ModelConfig::setSpeculativeDecodingModule__std::shared_ptr:SpeculativeDecodingModule:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a9aefc7d81ff2f1e4b9ec77e16145afbe"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">speculativeDecodingModule</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28setSpeculativeDecodingModuleERKNSt10shared_ptrI25SpeculativeDecodingModuleEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig30resetSpeculativeDecodingModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig30resetSpeculativeDecodingModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig30resetSpeculativeDecodingModuleEv"></span><span id="tensorrt_llm::runtime::ModelConfig::resetSpeculativeDecodingModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8fa60f44c0453831c73d10daef5da888"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">resetSpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig30resetSpeculativeDecodingModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig31enableSeamlessLookaheadDecodingE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig31enableSeamlessLookaheadDecodingE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig31enableSeamlessLookaheadDecodingE10SizeType32"></span><span id="tensorrt_llm::runtime::ModelConfig::enableSeamlessLookaheadDecoding__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a0636815cd95cc6c8fbb47f8d3195bc5a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enableSeamlessLookaheadDecoding</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftTokens</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig31enableSeamlessLookaheadDecodingE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig32disableSeamlessLookaheadDecodingEv">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig32disableSeamlessLookaheadDecodingEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig32disableSeamlessLookaheadDecodingEv"></span><span id="tensorrt_llm::runtime::ModelConfig::disableSeamlessLookaheadDecoding"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a9de9588bdfa6b49c43448d3af1e2d27b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableSeamlessLookaheadDecoding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig32disableSeamlessLookaheadDecodingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getKvDataTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig13getKvDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig13getKvDataTypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getKvDataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1acba08eede239343e16f5d5588df3c21d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getKvDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getKvDataTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18isTransformerBasedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18isTransformerBasedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18isTransformerBasedEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isTransformerBasedCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a22e6fb7c8e43e261c6d4a20b092a9e74"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTransformerBased</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18isTransformerBasedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12hasRnnConfigEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12hasRnnConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12hasRnnConfigEv"></span><span id="tensorrt_llm::runtime::ModelConfig::hasRnnConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a25e3be93fdc314a3ee7e3e4e9eeda670"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">hasRnnConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12hasRnnConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getRnnConfigEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12getRnnConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12getRnnConfigEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getRnnConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac0bfbddabe94a39760e15534aef6ea00"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE" title="tensorrt_llm::runtime::ModelConfig::RnnConfig"><span class="n"><span class="pre">RnnConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRnnConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getRnnConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12setRnnConfigERK9RnnConfig">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12setRnnConfigERK9RnnConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12setRnnConfigERK9RnnConfig"></span><span id="tensorrt_llm::runtime::ModelConfig::setRnnConfig__RnnConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a52903902f3cac457de86a80f27a4aa40"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setRnnConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE" title="tensorrt_llm::runtime::ModelConfig::RnnConfig"><span class="n"><span class="pre">RnnConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">rnnConfig</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setRnnConfigERK9RnnConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig10isRnnBasedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig10isRnnBasedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig10isRnnBasedEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isRnnBasedCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5e45d06e927ee447d35b7f6af188fb1d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isRnnBased</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig10isRnnBasedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getLayerTypesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig13getLayerTypesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig13getLayerTypesEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getLayerTypesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5837dd2a49d55f5c62364a6a6601af1d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="tensorrt_llm::runtime::ModelConfig::LayerType"><span class="n"><span class="pre">LayerType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getLayerTypes</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getLayerTypesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13setLayerTypesERKNSt6vectorI9LayerTypeEE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13setLayerTypesERKNSt6vectorI9LayerTypeEE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13setLayerTypesERKNSt6vectorI9LayerTypeEE"></span><span id="tensorrt_llm::runtime::ModelConfig::setLayerTypes__std::vector:LayerType:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2d5af32049283dfa801379c561dcd97f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLayerTypes</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="tensorrt_llm::runtime::ModelConfig::LayerType"><span class="n"><span class="pre">LayerType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">layerTypes</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13setLayerTypesERKNSt6vectorI9LayerTypeEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig26getSpeculativeDecodingModeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig26getSpeculativeDecodingModeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig26getSpeculativeDecodingModeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getSpeculativeDecodingModeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab21e510daf4bdf89d432ad0fae9fb7df"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSpeculativeDecodingMode</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig26getSpeculativeDecodingModeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLogitsDtypeEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14setLogitsDtypeEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14setLogitsDtypeEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::setLogitsDtype__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a521abb493b4b844a44ccd94121d2b9ed"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setLogitsDtype</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inputDtype</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLogitsDtypeEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLogitsDtypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig14getLogitsDtypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig14getLogitsDtypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getLogitsDtypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1adbf3366f2f80450af6a42bfa5f2942f7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLogitsDtype</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLogitsDtypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21setGemmAllReduceDtypeEN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21setGemmAllReduceDtypeEN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21setGemmAllReduceDtypeEN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::setGemmAllReduceDtype__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5b6283b91be75e2b774cd0c3e41220c0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setGemmAllReduceDtype</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inputDtype</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setGemmAllReduceDtypeEN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getGemmAllReduceDtypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig21getGemmAllReduceDtypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig21getGemmAllReduceDtypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getGemmAllReduceDtypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8021cd3b74d441b0ebf1914fd1ae00a1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGemmAllReduceDtype</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getGemmAllReduceDtypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseShapeInferenceEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20setUseShapeInferenceEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20setUseShapeInferenceEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setUseShapeInference__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af9020a3352a95c41af6ccf856fc63740"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setUseShapeInference</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">useShapeInference</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseShapeInferenceEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useShapeInferenceEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig17useShapeInferenceEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig17useShapeInferenceEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useShapeInferenceC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac3900be3c0184e7674b6ca9cf1c1e24f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useShapeInference</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useShapeInferenceEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getManageWeightsTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig20getManageWeightsTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig20getManageWeightsTypeEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getManageWeightsTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5ea16427c0ef9dc4e4cb3ab297400684"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE" title="tensorrt_llm::runtime::ModelConfig::ManageWeightsType"><span class="n"><span class="pre">ManageWeightsType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getManageWeightsType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getManageWeightsTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20setManageWeightsTypeEK17ManageWeightsType">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20setManageWeightsTypeEK17ManageWeightsType"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20setManageWeightsTypeEK17ManageWeightsType"></span><span id="tensorrt_llm::runtime::ModelConfig::setManageWeightsType__ManageWeightsTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab65ef4a60cd75d3e2f18c91486592a34"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setManageWeightsType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE" title="tensorrt_llm::runtime::ModelConfig::ManageWeightsType"><span class="n"><span class="pre">ManageWeightsType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">manageWeightType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setManageWeightsTypeEK17ManageWeightsType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getModelNameEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12getModelNameEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12getModelNameEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getModelNameC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a38ff64aeb32deaa59009048788ef7205"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getModelName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getModelNameEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12setModelNameERKNSt6stringE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12setModelNameERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12setModelNameERKNSt6stringE"></span><span id="tensorrt_llm::runtime::ModelConfig::setModelName__ssCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a466ea60c644630d431b4db7882123756"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setModelName</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelName</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setModelNameERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getNumKvHeadsPerLayerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig21getNumKvHeadsPerLayerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig21getNumKvHeadsPerLayerEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getNumKvHeadsPerLayerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2d5f312460c7acec54548c8530342f3a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getNumKvHeadsPerLayer</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getNumKvHeadsPerLayerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getNumKvHeadsPerLayerLocalRangeE10SizeType3210SizeType32b">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig31getNumKvHeadsPerLayerLocalRangeE10SizeType3210SizeType32b"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig31getNumKvHeadsPerLayerLocalRangeE10SizeType3210SizeType32b"></span><span id="tensorrt_llm::runtime::ModelConfig::getNumKvHeadsPerLayerLocalRange__SizeType32.SizeType32.bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a05ea49becb87faba3755e95a98484aac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">pair</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">const_iterator</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">const_iterator</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumKvHeadsPerLayerLocalRange</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isCrossAttention</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getNumKvHeadsPerLayerLocalRangeE10SizeType3210SizeType32b" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21setNumKvHeadsPerLayerERKNSt6vectorI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21setNumKvHeadsPerLayerERKNSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21setNumKvHeadsPerLayerERKNSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::runtime::ModelConfig::setNumKvHeadsPerLayer__std::vector:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad0711ba3755ba943f52a8bfee4adffcb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumKvHeadsPerLayer</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">headsPerLayer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setNumKvHeadsPerLayerERKNSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig26setNumKvHeadsPerCrossLayerERKNSt6vectorI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig26setNumKvHeadsPerCrossLayerERKNSt6vectorI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig26setNumKvHeadsPerCrossLayerERKNSt6vectorI10SizeType32EE"></span><span id="tensorrt_llm::runtime::ModelConfig::setNumKvHeadsPerCrossLayer__std::vector:SizeType32:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a59b7dba50c19d9e418a9816b668340e4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumKvHeadsPerCrossLayer</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">headsPerLayer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26setNumKvHeadsPerCrossLayerERKNSt6vectorI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getSumLocalKvHeadsE10SizeType3210SizeType32b">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18getSumLocalKvHeadsE10SizeType3210SizeType32b"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18getSumLocalKvHeadsE10SizeType3210SizeType32b"></span><span id="tensorrt_llm::runtime::ModelConfig::getSumLocalKvHeads__SizeType32.SizeType32.bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a639038d8052a0789c54b0d6c7f1ee040"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSumLocalKvHeads</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelismRank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isCrossAttention</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getSumLocalKvHeadsE10SizeType3210SizeType32b" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig19skipCrossAttnBlocksEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig19skipCrossAttnBlocksEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig19skipCrossAttnBlocksEv"></span><span id="tensorrt_llm::runtime::ModelConfig::skipCrossAttnBlocksCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a144f4f5b81ad7d6d42159033418f1a94"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">skipCrossAttnBlocks</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19skipCrossAttnBlocksEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig22setSkipCrossAttnBlocksEb">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig22setSkipCrossAttnBlocksEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig22setSkipCrossAttnBlocksEb"></span><span id="tensorrt_llm::runtime::ModelConfig::setSkipCrossAttnBlocks__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a44877e23b091833fcc848a92be8e53f1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSkipCrossAttnBlocks</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">skipCrossAttnBlocks</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22setSkipCrossAttnBlocksEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getNumLanguagesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig15getNumLanguagesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig15getNumLanguagesEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getNumLanguagesCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a1c8139ff257f6a0a06e9abba9c84190f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumLanguages</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getNumLanguagesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig18useLanguageAdapterEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig18useLanguageAdapterEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig18useLanguageAdapterEv"></span><span id="tensorrt_llm::runtime::ModelConfig::useLanguageAdapterCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a5be3306ef32fc18137159181656e4ab8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useLanguageAdapter</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18useLanguageAdapterEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15setNumLanguagesENSt8optionalI10SizeType32EE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15setNumLanguagesENSt8optionalI10SizeType32EE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15setNumLanguagesENSt8optionalI10SizeType32EE"></span><span id="tensorrt_llm::runtime::ModelConfig::setNumLanguages__std::optional:SizeType32:CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3b0caa514ac70bad83c7d6c2c06dac91"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumLanguages</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">numLanguages</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setNumLanguagesENSt8optionalI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig12isMultiModalEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig12isMultiModalEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig12isMultiModalEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isMultiModalC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab7573c40d0f633ffbb2fd008ce8ae5dc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isMultiModal</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12isMultiModalEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11ModelConfig9isWhisperEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11ModelConfig9isWhisperEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11ModelConfig9isWhisperEv"></span><span id="tensorrt_llm::runtime::ModelConfig::isWhisperC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1abe0bf7ebc4143e0e7e4bae67632e3127"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isWhisper</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig9isWhisperEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21KVCacheTypeFromStringENSt6stringE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21KVCacheTypeFromStringENSt6stringE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21KVCacheTypeFromStringENSt6stringE"></span><span id="tensorrt_llm::runtime::ModelConfig::KVCacheTypeFromString__ss"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1adc0f27d15df1f1467da047c129056d0b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType"><span class="n"><span class="pre">KVCacheType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">KVCacheTypeFromString</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">value</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21KVCacheTypeFromStringENSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig25getOptProfilesSplitPointsEv">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig25getOptProfilesSplitPointsEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig25getOptProfilesSplitPointsEv"></span><span id="tensorrt_llm::runtime::ModelConfig::getOptProfilesSplitPoints"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a63b9edc5318d4fbd754446da66ec93dc"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getOptProfilesSplitPoints</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig25getOptProfilesSplitPointsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig26kOPT_PROFILES_SPLIT_POINTSE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig26kOPT_PROFILES_SPLIT_POINTSE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig26kOPT_PROFILES_SPLIT_POINTSE"></span><span id="tensorrt_llm::runtime::ModelConfig::kOPT_PROFILES_SPLIT_POINTS__std::array"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a88507c9a7c6f52dc95eec39741eb8d47"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">array</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kOPT_PROFILES_SPLIT_POINTS</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">64</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">128</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">256</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">512</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1024</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26kOPT_PROFILES_SPLIT_POINTSE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig29kDEFAULT_NUM_TOKENS_PER_BLOCKE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig29kDEFAULT_NUM_TOKENS_PER_BLOCKE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig29kDEFAULT_NUM_TOKENS_PER_BLOCKE"></span><span id="tensorrt_llm::runtime::ModelConfig::kDEFAULT_NUM_TOKENS_PER_BLOCK__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a206f572f528fa3e31abc71080f1aa2cd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDEFAULT_NUM_TOKENS_PER_BLOCK</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">64</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig29kDEFAULT_NUM_TOKENS_PER_BLOCKE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig10mVocabSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig10mVocabSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig10mVocabSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mVocabSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a37a3d0383c8a5d366a6d914be237b971"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mVocabSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mVocabSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9mNbLayersE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9mNbLayersE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9mNbLayersE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNbLayers__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a86f9c01d713df2e5ac6dbf5c96e256ca"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNbLayers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mNbLayersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18mNbAttentionLayersE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18mNbAttentionLayersE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18mNbAttentionLayersE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNbAttentionLayers__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aff01f3f88389157b57553c5d73d3f8c6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNbAttentionLayers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mNbAttentionLayersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mNbRnnLayersE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mNbRnnLayersE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mNbRnnLayersE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNbRnnLayers__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a61c3f51a17a377b956834b6b716fb606"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNbRnnLayers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mNbRnnLayersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig8mNbHeadsE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig8mNbHeadsE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig8mNbHeadsE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNbHeads__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2c02a45e1219d6b00dbcf3637fcaf231"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNbHeads</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig8mNbHeadsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11mHiddenSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11mHiddenSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11mHiddenSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mHiddenSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2121e3fbad27dd5672a58c6214accaa0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mHiddenSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mHiddenSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mSizePerHeadE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mSizePerHeadE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mSizePerHeadE"></span><span id="tensorrt_llm::runtime::ModelConfig::mSizePerHead__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a21af1497012f9bddd323992c1a04d677"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSizePerHead</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mSizePerHeadE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9mDataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9mDataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9mDataTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mDataType__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1affb02d8d91bd93ae9ccb486bbdb2abfe"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mDataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseGptAttentionPluginE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig22mUseGptAttentionPluginE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig22mUseGptAttentionPluginE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseGptAttentionPlugin__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aad9e2f0b2d194a09bdb97fad5ea34cf8"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseGptAttentionPlugin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseGptAttentionPluginE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig23mUseGemmAllReducePluginE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig23mUseGemmAllReducePluginE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig23mUseGemmAllReducePluginE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseGemmAllReducePlugin__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a681bae7222920c3451198ff197754eae"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseGemmAllReducePlugin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23mUseGemmAllReducePluginE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig19mGemmAllReduceDtypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig19mGemmAllReduceDtypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig19mGemmAllReduceDtypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mGemmAllReduceDtype__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae01e3a0f31bc416b5bf8c3dc15e09b89"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGemmAllReduceDtype</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19mGemmAllReduceDtypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUseMambaConv1dPluginE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21mUseMambaConv1dPluginE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21mUseMambaConv1dPluginE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseMambaConv1dPlugin__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a36680ffd87ff6fcb1d04850d5e8bf7bf"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseMambaConv1dPlugin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUseMambaConv1dPluginE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mInputPackedE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mInputPackedE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mInputPackedE"></span><span id="tensorrt_llm::runtime::ModelConfig::mInputPacked__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a882a61ec0abd6731fe39e89b3842f22e"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInputPacked</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mInputPackedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11mPagedStateE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11mPagedStateE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11mPagedStateE"></span><span id="tensorrt_llm::runtime::ModelConfig::mPagedState__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8ecdf08d955ce84fb86d1a1b2e2ccf08"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPagedState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mPagedStateE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15mTokensPerBlockE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15mTokensPerBlockE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15mTokensPerBlockE"></span><span id="tensorrt_llm::runtime::ModelConfig::mTokensPerBlock__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa046f0031177bbcb63ce5897ebcddde2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTokensPerBlock</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15mTokensPerBlockE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig10mQuantModeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig10mQuantModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig10mQuantModeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mQuantMode__common::QuantMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a61b5f8cdf8fccb1c130d260033e2ff00"></span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">QuantMode</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mQuantMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mQuantModeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBatchSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13mMaxBatchSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13mMaxBatchSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxBatchSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a61a5e9d7334719dd85876b36bb6b15a2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBatchSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBatchSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBeamWidthE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13mMaxBeamWidthE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13mMaxBeamWidthE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxBeamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a1813511a528d48f49e9e8e3aa3c0352c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxBeamWidth</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBeamWidthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxInputLenE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mMaxInputLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mMaxInputLenE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxInputLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa93bc77b0e2f29dc570da55dc23e5da3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxInputLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxInputLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig15mMaxSequenceLenE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig15mMaxSequenceLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig15mMaxSequenceLenE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxSequenceLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1aa5aceb745909848d3a44e6d7d25e1154"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxSequenceLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15mMaxSequenceLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxNumTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13mMaxNumTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13mMaxNumTokensE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxNumTokens__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a28f632083f25f9916316be1ea4b7376b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxNumTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxNumTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21mComputeContextLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21mComputeContextLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21mComputeContextLogitsE"></span><span id="tensorrt_llm::runtime::ModelConfig::mComputeContextLogits__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a62f06f9d9223d6ce86537666aabab262"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mComputeContextLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mComputeContextLogitsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig24mComputeGenerationLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig24mComputeGenerationLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig24mComputeGenerationLogitsE"></span><span id="tensorrt_llm::runtime::ModelConfig::mComputeGenerationLogits__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8fc084e75b3e7163e1a55431e911e71f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mComputeGenerationLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24mComputeGenerationLogitsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13mModelVariantE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13mModelVariantE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13mModelVariantE"></span><span id="tensorrt_llm::runtime::ModelConfig::mModelVariant__ModelVariant"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad1b58680292bdf1913810e12f031fd09"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE" title="tensorrt_llm::runtime::ModelConfig::ModelVariant"><span class="n"><span class="pre">ModelVariant</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModelVariant</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mModelVariantE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig28mMaxPromptEmbeddingTableSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig28mMaxPromptEmbeddingTableSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig28mMaxPromptEmbeddingTableSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxPromptEmbeddingTableSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ad4bf7c9661e68d3b2711d90ff943fe83"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxPromptEmbeddingTableSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28mMaxPromptEmbeddingTableSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9mUseMropeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9mUseMropeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9mUseMropeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseMrope__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a532a93789582c93ce8da37d7672904a5"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseMrope</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mUseMropeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig22mMaxPositionEmbeddingsE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig22mMaxPositionEmbeddingsE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig22mMaxPositionEmbeddingsE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxPositionEmbeddings__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1abb4f2bc166989ca1ddfc7ae5485f7386"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxPositionEmbeddings</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mMaxPositionEmbeddingsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig19mRotaryEmbeddingDimE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig19mRotaryEmbeddingDimE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig19mRotaryEmbeddingDimE"></span><span id="tensorrt_llm::runtime::ModelConfig::mRotaryEmbeddingDim__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8f2487f2464369a6e503fefcf92d34b5"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRotaryEmbeddingDim</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19mRotaryEmbeddingDimE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mContextFMHAE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mContextFMHAE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mContextFMHAE"></span><span id="tensorrt_llm::runtime::ModelConfig::mContextFMHA__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a9571d6946c7b8a64d6e69bf6746ff639"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mContextFMHA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mContextFMHAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig17mPagedContextFMHAE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig17mPagedContextFMHAE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig17mPagedContextFMHAE"></span><span id="tensorrt_llm::runtime::ModelConfig::mPagedContextFMHA__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3f6ec1842133c50530dd26c9faaf7a52"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPagedContextFMHA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17mPagedContextFMHAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig16mPpReduceScatterE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig16mPpReduceScatterE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig16mPpReduceScatterE"></span><span id="tensorrt_llm::runtime::ModelConfig::mPpReduceScatter__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ab258eb7446c0efa12a53d655e97ec86c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPpReduceScatter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16mPpReduceScatterE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14mUseLoraPluginE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14mUseLoraPluginE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14mUseLoraPluginE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseLoraPlugin__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3be94f3bdf5c5e2e08482f98537d3d90"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseLoraPlugin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mUseLoraPluginE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLoraModulesE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mLoraModulesE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mLoraModulesE"></span><span id="tensorrt_llm::runtime::ModelConfig::mLoraModules__std::vector:LoraModule:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a34b85b164e6d90afaa45ff133b6c8c03"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLoraModules</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLoraModulesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMlpHiddenSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14mMlpHiddenSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14mMlpHiddenSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMlpHiddenSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a54508516a416b2ae44d1d82253ad9678"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMlpHiddenSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMlpHiddenSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxLoraRankE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mMaxLoraRankE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mMaxLoraRankE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxLoraRank__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a78d71eb5426538c3e343cbc92fbbdabe"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxLoraRank</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxLoraRankE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig10mRnnConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig10mRnnConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig10mRnnConfigE"></span><span id="tensorrt_llm::runtime::ModelConfig::mRnnConfig__std::optional:RnnConfig:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a2376ed37b3e21ced6c540e83aba6ca60"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE" title="tensorrt_llm::runtime::ModelConfig::RnnConfig"><span class="n"><span class="pre">RnnConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRnnConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mRnnConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mKVCacheTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mKVCacheTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mKVCacheTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mKVCacheType__KVCacheType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a3ec8c245cfa56077d52e97fe55d6e75d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType"><span class="n"><span class="pre">KVCacheType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mKVCacheType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType"><span class="n"><span class="pre">KVCacheType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE" title="tensorrt_llm::runtime::ModelConfig::KVCacheType::kCONTINUOUS"><span class="n"><span class="pre">kCONTINUOUS</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mKVCacheTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMaxEncoderLenE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig14mMaxEncoderLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig14mMaxEncoderLenE"></span><span id="tensorrt_llm::runtime::ModelConfig::mMaxEncoderLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8c4cdc68888b85fbd943e5edeb7922c7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxEncoderLen</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMaxEncoderLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18mEncoderHiddenSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18mEncoderHiddenSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18mEncoderHiddenSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mEncoderHiddenSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a431ae6b9afe40b5bd4d2206a5cc7840d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEncoderHiddenSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mEncoderHiddenSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseCrossAttentionE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18mUseCrossAttentionE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18mUseCrossAttentionE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseCrossAttention__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1abf9500bc966e156bf0d476836d2b572a"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseCrossAttention</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseCrossAttentionE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUsePositionEmbeddingE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig21mUsePositionEmbeddingE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig21mUsePositionEmbeddingE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUsePositionEmbedding__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ac1492ed33343b577717489050a8a4b5c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUsePositionEmbedding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUsePositionEmbeddingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseTokenTypeEmbeddingE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig22mUseTokenTypeEmbeddingE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig22mUseTokenTypeEmbeddingE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseTokenTypeEmbedding__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1af6dc99e292a0518d5fcd1e221786d603"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseTokenTypeEmbedding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseTokenTypeEmbeddingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig11mLayerTypesE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig11mLayerTypesE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig11mLayerTypesE"></span><span id="tensorrt_llm::runtime::ModelConfig::mLayerTypes__std::vector:LayerType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a147a322e201414cb6fc34d0ed8706e68"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE" title="tensorrt_llm::runtime::ModelConfig::LayerType"><span class="n"><span class="pre">LayerType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLayerTypes</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mLayerTypesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig26mSpeculativeDecodingModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig26mSpeculativeDecodingModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig26mSpeculativeDecodingModuleE"></span><span id="tensorrt_llm::runtime::ModelConfig::mSpeculativeDecodingModule__std::shared_ptr:SpeculativeDecodingModule:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a6835321ce94e9fd421c391f1b5130a61"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSpeculativeDecodingModule</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26mSpeculativeDecodingModuleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig24mSpeculativeDecodingModeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig24mSpeculativeDecodingModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig24mSpeculativeDecodingModeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mSpeculativeDecodingMode__SpeculativeDecodingMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a8972453f4a38857ff67333c8736b36a3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSpeculativeDecodingMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24mSpeculativeDecodingModeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLogitsDtypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig12mLogitsDtypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig12mLogitsDtypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mLogitsDtype__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a031d0a099cfde1aea2f1181dd135f084"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mLogitsDtype</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLogitsDtypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseShapeInferenceE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18mUseShapeInferenceE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18mUseShapeInferenceE"></span><span id="tensorrt_llm::runtime::ModelConfig::mUseShapeInference__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1ae913a073cfb1a11c28886cb567c5900b"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUseShapeInference</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseShapeInferenceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig18mManageWeightsTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig18mManageWeightsTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig18mManageWeightsTypeE"></span><span id="tensorrt_llm::runtime::ModelConfig::mManageWeightsType__ManageWeightsType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a64d9addbd71e0ed7d43db7b035ed5c11"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE" title="tensorrt_llm::runtime::ModelConfig::ManageWeightsType"><span class="n"><span class="pre">ManageWeightsType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mManageWeightsType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mManageWeightsTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig10mModelNameE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig10mModelNameE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig10mModelNameE"></span><span id="tensorrt_llm::runtime::ModelConfig::mModelName__ss"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a040336df17e786e007f91dac8d0a5f4c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModelName</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mModelNameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig28mNumKvHeadsPerAttentionLayerE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig28mNumKvHeadsPerAttentionLayerE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig28mNumKvHeadsPerAttentionLayerE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNumKvHeadsPerAttentionLayer__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a476d2ac4959e8974ea5779c9b66387a3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumKvHeadsPerAttentionLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28mNumKvHeadsPerAttentionLayerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig33mNumKvHeadsPerCrossAttentionLayerE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig33mNumKvHeadsPerCrossAttentionLayerE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig33mNumKvHeadsPerCrossAttentionLayerE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNumKvHeadsPerCrossAttentionLayer__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a6295c0b948ef1c4220c1248a7f54b9f2"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumKvHeadsPerCrossAttentionLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig33mNumKvHeadsPerCrossAttentionLayerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig20mSkipCrossAttnBlocksE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig20mSkipCrossAttnBlocksE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig20mSkipCrossAttnBlocksE"></span><span id="tensorrt_llm::runtime::ModelConfig::mSkipCrossAttnBlocks__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1a4201343e4746e9f733c5b82c0a395aa3"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSkipCrossAttnBlocks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20mSkipCrossAttnBlocksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig13mNumLanguagesE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig13mNumLanguagesE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig13mNumLanguagesE"></span><span id="tensorrt_llm::runtime::ModelConfig::mNumLanguages__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ModelConfig_1abb2b95ebcf9f45ffc0443d08cdc20123"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumLanguages</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mNumLanguagesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfigE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RnnConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig9stateSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfig9stateSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfig9stateSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig::stateSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig_1aa48db3c2a457acac081d0464eb810d10"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stateSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig9stateSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig10convKernelE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfig10convKernelE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfig10convKernelE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig::convKernel__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig_1a95f342da763c9eccf41f82022d6aaece"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">convKernel</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig10convKernelE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig13rnnHiddenSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfig13rnnHiddenSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfig13rnnHiddenSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig::rnnHiddenSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig_1a563fdfb3179e823cebf6a29bfa79af6f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">rnnHiddenSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig13rnnHiddenSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig11rnnHeadSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfig11rnnHeadSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfig11rnnHeadSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig::rnnHeadSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig_1a224165c5e8b9a336c97c0630ef87b5af"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">rnnHeadSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig11rnnHeadSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig14rnnConvDimSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime11ModelConfig9RnnConfig14rnnConvDimSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11ModelConfig9RnnConfig14rnnConvDimSizeE"></span><span id="tensorrt_llm::runtime::ModelConfig::RnnConfig::rnnConvDimSize__SizeType32"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1ModelConfig_1_1RnnConfig_1af3e1e4f57fd7ce7585b9d6466125c6dd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">rnnConvDimSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig14rnnConvDimSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="loracachepagemanagerconfig-h">
<h2>loraCachePageManagerConfig.h<a class="headerlink" href="#loracachepagemanagerconfig-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK26LoraCachePageManagerConfig">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERK26LoraCachePageManagerConfig"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERK26LoraCachePageManagerConfig"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.LoraCachePageManagerConfigCR"></span><span class="target" id="loraCachePageManagerConfig_8h_1a02fbd0e78a87e57947a3d24b50f5141a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">os</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">c</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK26LoraCachePageManagerConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9to_stringERK26LoraCachePageManagerConfig">
<span id="_CPPv3N12tensorrt_llm7runtime9to_stringERK26LoraCachePageManagerConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime9to_stringERK26LoraCachePageManagerConfig"></span><span id="tensorrt_llm::runtime::to_string__LoraCachePageManagerConfigCR"></span><span class="target" id="loraCachePageManagerConfig_8h_1ab560cb107debf46c244846e86a6af35a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">to_string</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="tensorrt_llm::runtime::LoraCachePageManagerConfig"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">c</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9to_stringERK26LoraCachePageManagerConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfigE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;loraCachePageManagerConfig.h&gt;</em></div>
<p>Configuration for <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCachePageManager"><span class="std std-ref">LoraCachePageManager</span></a></p>
<p>See <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LoraCache"><span class="std std-ref">LoraCache</span></a> docs for description of pages, slots, and page blocks. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig26LoraCachePageManagerConfigEN7runtime10MemoryTypeEN8nvinfer18DataTypeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig26LoraCachePageManagerConfigEN7runtime10MemoryTypeEN8nvinfer18DataTypeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig26LoraCachePageManagerConfigEN7runtime10MemoryTypeEN8nvinfer18DataTypeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::LoraCachePageManagerConfig__runtime::MemoryType.nvinfer1::DataType.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a6e1aff651ea70a7fb83e92ee5b6973bf"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraCachePageManagerConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">memType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dType</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">totalNumPages</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxPagesPerBlock</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">slotsPerPage</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pageWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCopyStreams</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig26LoraCachePageManagerConfigEN7runtime10MemoryTypeEN8nvinfer18DataTypeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getMemoryTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getMemoryTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getMemoryTypeEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getMemoryTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a1d555270b7def499025cefa19b7abd03"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMemoryType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getMemoryTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setMemoryTypeERKN7runtime10MemoryTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setMemoryTypeERKN7runtime10MemoryTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setMemoryTypeERKN7runtime10MemoryTypeE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setMemoryType__runtime::MemoryTypeCRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a15aa48ad133fb36443123a0e38796173"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMemoryType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">memoryType</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setMemoryTypeERKN7runtime10MemoryTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig11getDataTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig11getDataTypeEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getDataTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a5b8569adff45e7ba8fae9147bf42a4a9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11setDataTypeERKN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig11setDataTypeERKN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig11setDataTypeERKN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setDataType__nvinfer1::DataTypeCRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1ac6c9e32d3645e715c9114aabb0a3aed1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setDataType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11setDataTypeERKN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig16getTotalNumPagesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig16getTotalNumPagesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig16getTotalNumPagesEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getTotalNumPagesCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1ac78c314b429c387bf16594e813aec5fd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTotalNumPages</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig16getTotalNumPagesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setTotalNumPageERK10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setTotalNumPageERK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setTotalNumPageERK10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setTotalNumPage__SizeType32CRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1afb9b7258d161eb6069a1cf568ac3e2f3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setTotalNumPage</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">totalNumPages</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setTotalNumPageERK10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig19getMaxPagesPerBlockEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig19getMaxPagesPerBlockEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig19getMaxPagesPerBlockEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getMaxPagesPerBlockCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1abcbd157aba25cb7c6b71961077b3ac4a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPagesPerBlock</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig19getMaxPagesPerBlockEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig19setMaxPagesPerBlockERK10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig19setMaxPagesPerBlockERK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig19setMaxPagesPerBlockERK10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setMaxPagesPerBlock__SizeType32CRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1aa3f4219a0ddab455d727fe54fc834999"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxPagesPerBlock</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">maxPagesPerBlock</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig19setMaxPagesPerBlockERK10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig15getSlotsPerPageEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig15getSlotsPerPageEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig15getSlotsPerPageEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getSlotsPerPageCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a0cb73d5d68b2ff2aa05e55c65aabb6eb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSlotsPerPage</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig15getSlotsPerPageEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setSlotsPerPageERK10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setSlotsPerPageERK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setSlotsPerPageERK10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setSlotsPerPage__SizeType32CRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a2741800c4c8b6b550af4894410acfecb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setSlotsPerPage</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">slotsPerPage</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setSlotsPerPageERK10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig12getPageWidthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig12getPageWidthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig12getPageWidthEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getPageWidthCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1add88d63cdf4bd8a6220a2662709b6e4b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPageWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig12getPageWidthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig12setPageWidthERK10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig12setPageWidthERK10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig12setPageWidthERK10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setPageWidth__SizeType32CRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1ac79621f57044a1cb648067d9e0594347"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPageWidth</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">pageWidth</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig12setPageWidthERK10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getInitToZeroEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getInitToZeroEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getInitToZeroEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getInitToZeroCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a4550e479df29f722b2c75e8b7614de45"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getInitToZero</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getInitToZeroEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setInitToZeroEb">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setInitToZeroEb"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setInitToZeroEb"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setInitToZero__bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a7f940837905e42adec2d6ed678a3677f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setInitToZero</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">initToZero</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setInitToZeroEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig17getNumCopyStreamsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime26LoraCachePageManagerConfig17getNumCopyStreamsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime26LoraCachePageManagerConfig17getNumCopyStreamsEv"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::getNumCopyStreamsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a3928d560911fb6b831a1ab11b5ae60dc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumCopyStreams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig17getNumCopyStreamsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17setNumCopyStreamsE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig17setNumCopyStreamsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig17setNumCopyStreamsE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::setNumCopyStreams__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a0a8555fa27f30b63a6d13fd4698303d6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setNumCopyStreams</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCopyStreams</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17setNumCopyStreamsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mMemoryTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mMemoryTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mMemoryTypeE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mMemoryType__runtime::MemoryType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a20d70126704e1f716dfcaf06da538b86"></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMemoryType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mMemoryTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig9mDataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig9mDataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig9mDataTypeE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mDataType__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a016b0e9c25c6ee1587448dee9059b809"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig9mDataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig14mTotalNumPagesE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig14mTotalNumPagesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig14mTotalNumPagesE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mTotalNumPages__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a1517a8ee5accef3c6d7b88dbb6766ce0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTotalNumPages</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig14mTotalNumPagesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17mMaxPagesPerBlockE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig17mMaxPagesPerBlockE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig17mMaxPagesPerBlockE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mMaxPagesPerBlock__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a0f9187333756785772166e1302f3c4da"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxPagesPerBlock</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17mMaxPagesPerBlockE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13mSlotsPerPageE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig13mSlotsPerPageE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig13mSlotsPerPageE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mSlotsPerPage__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a4990fcbf90a213977985fbd41ac2f22c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mSlotsPerPage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13mSlotsPerPageE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig10mPageWidthE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig10mPageWidthE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig10mPageWidthE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mPageWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a57b18ff225e47d5e1f41c9962c85cad9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPageWidth</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig10mPageWidthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15mNumCopyStreamsE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig15mNumCopyStreamsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig15mNumCopyStreamsE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mNumCopyStreams__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a5abcc91f11d68bcbb50ba133ece478b6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumCopyStreams</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15mNumCopyStreamsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mInitToZeroE">
<span id="_CPPv3N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mInitToZeroE"></span><span id="_CPPv2N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mInitToZeroE"></span><span id="tensorrt_llm::runtime::LoraCachePageManagerConfig::mInitToZero__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraCachePageManagerConfig_1a010768dca591da9134e770156e44a141"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInitToZero</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mInitToZeroE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="worldconfig-h">
<h2>worldConfig.h<a class="headerlink" href="#worldconfig-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::WorldConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">WorldConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig11WorldConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalINSt6vectorI10SizeType32EEEEb">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig11WorldConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalINSt6vectorI10SizeType32EEEEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig11WorldConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalINSt6vectorI10SizeType32EEEEb"></span><span id="tensorrt_llm::runtime::WorldConfig::WorldConfig__SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.std::optional:std::vector:SizeType32::CR.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a541b7728feacf60f717e9379b11fc3da"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">WorldConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensorParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">contextParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">rank</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gpusPerNode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE" title="tensorrt_llm::runtime::WorldConfig::kDefaultGpusPerNode"><span class="n"><span class="pre">kDefaultGpusPerNode</span></span></a></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">deviceIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableAttentionDP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig11WorldConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalINSt6vectorI10SizeType32EEEEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig7getSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig7getSizeEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a8b9d102af9689ecca7b6e9924ca955a2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig20getTensorParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig20getTensorParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig20getTensorParallelismEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getTensorParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1ae5022fa448a9d76e460b1a255d47c9e3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTensorParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig20getTensorParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig16isTensorParallelEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig16isTensorParallelEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig16isTensorParallelEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isTensorParallelCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a00a72e2f83f447679b12024100e2bd51"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isTensorParallel</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig16isTensorParallelEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getPipelineParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig22getPipelineParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig22getPipelineParallelismEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getPipelineParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1ac69e3f6afd55e830b76b6a39a14481cd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPipelineParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getPipelineParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig18isPipelineParallelEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig18isPipelineParallelEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig18isPipelineParallelEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isPipelineParallelCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a26eaecd483304e8df407068905d9123c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isPipelineParallel</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig18isPipelineParallelEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getContextParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig21getContextParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig21getContextParallelismEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getContextParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a004b171a1af6b36bcb45df247c77485a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getContextParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig17isContextParallelEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig17isContextParallelEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig17isContextParallelEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isContextParallelCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a7258b051cb4ac27b5a99e5999467c733"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isContextParallel</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig17isContextParallelEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig7getRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig7getRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1af1cb5b83608c3da757e7dbe2b1e5597a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig14getGpusPerNodeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig14getGpusPerNodeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig14getGpusPerNodeEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getGpusPerNodeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a44a3e7694a2c357f5b5d63e5964cfcb2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpusPerNode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig14getGpusPerNodeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig15getGpusPerGroupEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig15getGpusPerGroupEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig15getGpusPerGroupEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getGpusPerGroupC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1adcd82f3f12d0fa200af350aa7e6c03fc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpusPerGroup</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig15getGpusPerGroupEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig9getDeviceEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig9getDeviceEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig9getDeviceEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getDeviceC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a0e1c32dce89cf5bb8a0c6442254b77aa"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDevice</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig9getDeviceEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getDeviceOfE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig11getDeviceOfE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig11getDeviceOfE10SizeType32"></span><span id="tensorrt_llm::runtime::WorldConfig::getDeviceOf__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1aa10799e03062dbc43bba2c25136ebf74"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDeviceOf</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">rank</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getDeviceOfE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getPipelineParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig23getPipelineParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig23getPipelineParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getPipelineParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a4edd655c3bd2758d67f0171d77e54f5d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPipelineParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getPipelineParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getTensorParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig21getTensorParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig21getTensorParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getTensorParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a63c6d87c37aadcd07700dd935b4a91e5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTensorParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getTensorParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getContextParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig22getContextParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig22getContextParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getContextParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1aa257d2b25d50ee74832f93c179b9ee41"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getContextParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig12getLocalRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig12getLocalRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig12getLocalRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getLocalRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1af7740d5c7afd1ba7a98f4b2e0f481838"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLocalRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig12getLocalRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getNodeRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig11getNodeRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig11getNodeRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getNodeRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a4e5ed903b009aee7f656931e4902c8ce"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNodeRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getNodeRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig13getNodeRankOfE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig13getNodeRankOfE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig13getNodeRankOfE10SizeType32"></span><span id="tensorrt_llm::runtime::WorldConfig::getNodeRankOf__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1ad0c5b7241276daca31e02c9305ea7fa2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNodeRankOf</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">rank</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig13getNodeRankOfE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig27isFirstPipelineParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig27isFirstPipelineParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig27isFirstPipelineParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isFirstPipelineParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a23a7f6bb812b6d0e60325e91c14cb2e0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isFirstPipelineParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig27isFirstPipelineParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isLastPipelineParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig26isLastPipelineParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig26isLastPipelineParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isLastPipelineParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a86b29f5ea72282f86f8af979edb6c3e7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLastPipelineParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isLastPipelineParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Is my rank the last rank in its pipeline? </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig25isFirstTensorParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig25isFirstTensorParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig25isFirstTensorParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isFirstTensorParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a00d09d798d8301bb87dd364f7a47193f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isFirstTensorParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig25isFirstTensorParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isFirstContextParallelRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig26isFirstContextParallelRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig26isFirstContextParallelRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::isFirstContextParallelRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a1134cb4738755b321c00e886ab716ac2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isFirstContextParallelRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isFirstContextParallelRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getLastRankEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig11getLastRankEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig11getLastRankEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getLastRankCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a8cf59e9a62e4801a2ae25f3b0cbc2e89"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getLastRank</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getLastRankEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig17enableAttentionDPEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig17enableAttentionDPEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig17enableAttentionDPEv"></span><span id="tensorrt_llm::runtime::WorldConfig::enableAttentionDPCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a7757e8af26edaced44f283fec7f85430"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enableAttentionDP</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig17enableAttentionDPEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig24getPipelineParallelGroupEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig24getPipelineParallelGroupEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig24getPipelineParallelGroupEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getPipelineParallelGroupC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1ad3ce7871a43bf0bf6ca72346b6605e02"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPipelineParallelGroup</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig24getPipelineParallelGroupEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getTensorParallelGroupEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig22getTensorParallelGroupEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig22getTensorParallelGroupEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getTensorParallelGroupC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1aac48fc5479138ba138634326ef49ed01"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTensorParallelGroup</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getTensorParallelGroupEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getContextParallelGroupEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig23getContextParallelGroupEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig23getContextParallelGroupEv"></span><span id="tensorrt_llm::runtime::WorldConfig::getContextParallelGroupC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1afbd5f464ff91ee9ca154b6c7d3b5447a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextParallelGroup</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getContextParallelGroupEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11WorldConfig14validMpiConfigEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11WorldConfig14validMpiConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11WorldConfig14validMpiConfigEv"></span><span id="tensorrt_llm::runtime::WorldConfig::validMpiConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a92e7fa800262ca7d7ca08f2705d30626"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">validMpiConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig14validMpiConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig3mpiE10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEEb">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig3mpiE10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEEb"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig3mpiE10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEEb"></span><span id="tensorrt_llm::runtime::WorldConfig::mpi__SizeType32.std::optional:SizeType32:.std::optional:SizeType32:.std::optional:SizeType32:.std::optional:std::vector:SizeType32::CR.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a4d1ccaa9346374229e19553ab72089ad"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mpi</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gpusPerNode</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE" title="tensorrt_llm::runtime::WorldConfig::kDefaultGpusPerNode"><span class="n"><span class="pre">kDefaultGpusPerNode</span></span></a></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">tensorParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">contextParallelism</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">deviceIds</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enableAttentionDP</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig3mpiE10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE"></span><span id="tensorrt_llm::runtime::WorldConfig::kDefaultGpusPerNode__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a947b944a0ba919cf264b2f40d6e88fe1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDefaultGpusPerNode</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig18mTensorParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig18mTensorParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig18mTensorParallelismE"></span><span id="tensorrt_llm::runtime::WorldConfig::mTensorParallelism__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1aa7b502999329a2e6c0befbec8bb391d7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTensorParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig18mTensorParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig20mPipelineParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig20mPipelineParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig20mPipelineParallelismE"></span><span id="tensorrt_llm::runtime::WorldConfig::mPipelineParallelism__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1af25b064d4e53a41f5c73ad2c2e7798b9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPipelineParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig20mPipelineParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig19mContextParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig19mContextParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig19mContextParallelismE"></span><span id="tensorrt_llm::runtime::WorldConfig::mContextParallelism__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1aac24e266668057de079b5cf50d9df978"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mContextParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19mContextParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig5mRankE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig5mRankE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig5mRankE"></span><span id="tensorrt_llm::runtime::WorldConfig::mRank__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a5f0fde85e5fe37245b4f8e544910dd29"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRank</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig5mRankE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig12mGpusPerNodeE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig12mGpusPerNodeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig12mGpusPerNodeE"></span><span id="tensorrt_llm::runtime::WorldConfig::mGpusPerNode__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a2db2a3ff84174617be9b1de7833f6792"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGpusPerNode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig12mGpusPerNodeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig18mEnableAttentionDPE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig18mEnableAttentionDPE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig18mEnableAttentionDPE"></span><span id="tensorrt_llm::runtime::WorldConfig::mEnableAttentionDP__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a018ea8d84caaeb997132f694a87bc005"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnableAttentionDP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig18mEnableAttentionDPE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11WorldConfig10mDeviceIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime11WorldConfig10mDeviceIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime11WorldConfig10mDeviceIdsE"></span><span id="tensorrt_llm::runtime::WorldConfig::mDeviceIds__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1WorldConfig_1a4e6848ca14ea58630295ffb14c365e39"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDeviceIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig10mDeviceIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="eaglemodule-h">
<h2>eagleModule.h<a class="headerlink" href="#eaglemodule-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModuleE"></span><span id="tensorrt_llm::runtime::EagleModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleModule</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModuleE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleE10SizeType3210SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModule11EagleModuleE10SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModule11EagleModuleE10SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::EagleModule::EagleModule__SizeType32.SizeType32.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1af166e0eb300764de18a9663596fcfa88"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftPathLen</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDecodingDraftTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numTransformersLayer</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNonLeafNodesPerLayer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleE10SizeType3210SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModule11EagleModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModule11EagleModuleEv"></span><span id="tensorrt_llm::runtime::EagleModule::EagleModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1a6e4abb448b27bf418e16d27880b2d838"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11EagleModule22getDefaultEagleChoicesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11EagleModule22getDefaultEagleChoicesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11EagleModule22getDefaultEagleChoicesEv"></span><span id="tensorrt_llm::runtime::EagleModule::getDefaultEagleChoicesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1a43c8ebb727e67f132e1a92eb699e56d4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="tensorrt_llm::executor::EagleChoices"><span class="n"><span class="pre">EagleChoices</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDefaultEagleChoices</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule22getDefaultEagleChoicesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11EagleModule23getNumTransformerLayersEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11EagleModule23getNumTransformerLayersEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11EagleModule23getNumTransformerLayersEv"></span><span id="tensorrt_llm::runtime::EagleModule::getNumTransformerLayersC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1a64dd03e74f7ac370b1a91b2258ca5459"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumTransformerLayers</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule23getNumTransformerLayersEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11EagleModule26getMaxNonLeafNodesPerLayerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime11EagleModule26getMaxNonLeafNodesPerLayerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime11EagleModule26getMaxNonLeafNodesPerLayerEv"></span><span id="tensorrt_llm::runtime::EagleModule::getMaxNonLeafNodesPerLayerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1a974f062f32ef741385ef58cbcef34107"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxNonLeafNodesPerLayer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule26getMaxNonLeafNodesPerLayerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModule21mNumTransformersLayerE">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModule21mNumTransformersLayerE"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModule21mNumTransformersLayerE"></span><span id="tensorrt_llm::runtime::EagleModule::mNumTransformersLayer__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1ac1aeec70a05e79aa9f6e8b49650fc658"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mNumTransformersLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule21mNumTransformersLayerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModule24mMaxNonLeafNodesPerLayerE">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModule24mMaxNonLeafNodesPerLayerE"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModule24mMaxNonLeafNodesPerLayerE"></span><span id="tensorrt_llm::runtime::EagleModule::mMaxNonLeafNodesPerLayer__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1acfc7f0572036f098de12c829c1be5d5f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxNonLeafNodesPerLayer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule24mMaxNonLeafNodesPerLayerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11EagleModule20mDefaultEagleChoicesE">
<span id="_CPPv3N12tensorrt_llm7runtime11EagleModule20mDefaultEagleChoicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime11EagleModule20mDefaultEagleChoicesE"></span><span id="tensorrt_llm::runtime::EagleModule::mDefaultEagleChoices__executor::EagleChoices"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleModule_1a1623f61ca148202804add817bb0ef6ce"></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12EagleChoicesE" title="tensorrt_llm::executor::EagleChoices"><span class="n"><span class="pre">EagleChoices</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDefaultEagleChoices</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule20mDefaultEagleChoicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="decodinginput-h">
<h2>decodingInput.h<a class="headerlink" href="#decodinginput-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInputE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInputE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInputE"></span><span id="tensorrt_llm::runtime::DecodingInput"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingInput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;decodingInput.h&gt;</em></div>
<p>Represents the inputs to the decoder. </p>
<p>This input type is assumed immutable. It represents whatever the decoder received initially, and can always be referred to as such. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a051f24825db26577ef03a898c41ee9a0"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a0e0863b2f0681e5b61953b61b2b072ee"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13DecodingInputE10SizeType3210SizeType3210SizeType3210SizeType3214TensorConstPtr9TensorPtr14TensorConstPtr">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13DecodingInputE10SizeType3210SizeType3210SizeType3210SizeType3214TensorConstPtr9TensorPtr14TensorConstPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13DecodingInputE10SizeType3210SizeType3210SizeType3210SizeType3214TensorConstPtr9TensorPtr14TensorConstPtr"></span><span id="tensorrt_llm::runtime::DecodingInput::DecodingInput__SizeType32.SizeType32.SizeType32.SizeType32.TensorConstPtr.TensorPtr.TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1af1901688d26b4be46f628c08d80f447e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingInput</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindow</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">logits</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">endIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13DecodingInputE10SizeType3210SizeType3210SizeType3210SizeType3214TensorConstPtr9TensorPtr14TensorConstPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput4stepE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput4stepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput4stepE"></span><span id="tensorrt_llm::runtime::DecodingInput::step__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a41c49960bea968b4c4e6b7d073c57769"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">step</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput4stepE" title="Link to this definition">#</a><br /></dt>
<dd><p>Mandatory parameters The index of the decoding step we are on. Only used in Python runtime </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput9maxLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput9maxLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput9maxLengthE"></span><span id="tensorrt_llm::runtime::DecodingInput::maxLength__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1abef240110c77063b264d9def9ae87706"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9maxLengthE" title="Link to this definition">#</a><br /></dt>
<dd><p>The maximum number of tokens to decode. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput18maxAttentionWindowE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput18maxAttentionWindowE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput18maxAttentionWindowE"></span><span id="tensorrt_llm::runtime::DecodingInput::maxAttentionWindow__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a54ab119d37f9a33cd54c4f9df3db6423"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxAttentionWindow</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput18maxAttentionWindowE" title="Link to this definition">#</a><br /></dt>
<dd><p>The maximum length of the attention window to consider while decoding. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15sinkTokenLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15sinkTokenLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15sinkTokenLengthE"></span><span id="tensorrt_llm::runtime::DecodingInput::sinkTokenLength__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a851a4ee559af06eeb0493627d3b8a57f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">sinkTokenLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15sinkTokenLengthE" title="Link to this definition">#</a><br /></dt>
<dd><p>The number of tokens to use as attention sinks, <a class="reference external" href="https://arxiv.org/html/2309.17453v3">https://arxiv.org/html/2309.17453v3</a>. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput9batchSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput9batchSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput9batchSizeE"></span><span id="tensorrt_llm::runtime::DecodingInput::batchSize__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ab57b39faa8bcf0aa3787a581772e97c3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9batchSizeE" title="Link to this definition">#</a><br /></dt>
<dd><p>The number of samples in the batch. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput10beamWidthsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput10beamWidthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput10beamWidthsE"></span><span id="tensorrt_llm::runtime::DecodingInput::beamWidths__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ab7a785e61f52dad2103657b7bff74b90"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">beamWidths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput10beamWidthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>The beam widths of each request, [batchSize]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15maxStopWordsLenE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15maxStopWordsLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15maxStopWordsLenE"></span><span id="tensorrt_llm::runtime::DecodingInput::maxStopWordsLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a88be1c6c33b42189c86ae0135d042531"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxStopWordsLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15maxStopWordsLenE" title="Link to this definition">#</a><br /></dt>
<dd><p>The maximum value in the <code class="docutils literal notranslate"><span class="pre">stopWordsLens</span></code> tensor. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput14maxBadWordsLenE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput14maxBadWordsLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput14maxBadWordsLenE"></span><span id="tensorrt_llm::runtime::DecodingInput::maxBadWordsLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1af0e5d6ebbb1e5dc5fed3ae6c6ac4ca2e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxBadWordsLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14maxBadWordsLenE" title="Link to this definition">#</a><br /></dt>
<dd><p>The maximum value in the <code class="docutils literal notranslate"><span class="pre">badWordsLens</span></code> tensor. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput6logitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput6logitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput6logitsE"></span><span id="tensorrt_llm::runtime::DecodingInput::logits__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a57cf6d318b6692e9fad94d5cfae221a3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput6logitsE" title="Link to this definition">#</a><br /></dt>
<dd><p>The output of the model forward computation, a probability distribution over the vocabulary [batchSize, beamWidth, vocabSizePadded] on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput9logitsVecE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput9logitsVecE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput9logitsVecE"></span><span id="tensorrt_llm::runtime::DecodingInput::logitsVec__std::optional:std::vector:TensorConstPtr::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1aeeff0bf4b6c1fbbbb314cb7facbb66fc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logitsVec</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9logitsVecE" title="Link to this definition">#</a><br /></dt>
<dd><p>Another view on the logits, [batchSize][beamWidth, vocabSizePadded] on gpu. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput6endIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput6endIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput6endIdsE"></span><span id="tensorrt_llm::runtime::DecodingInput::endIds__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1aa493d476a79110129048fe61ba343b0f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">endIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput6endIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>The end ids, [batchSize * beamWidth] on gpu. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput10batchSlotsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput10batchSlotsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput10batchSlotsE"></span><span id="tensorrt_llm::runtime::DecodingInput::batchSlots__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a6eb7e6db9122e600018d2ab58a8647b0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput10batchSlotsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Address map of the linear batch id to to the seq slots, [batchSize] on pinned, int32_t. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13finishReasonsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13finishReasonsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13finishReasonsE"></span><span id="tensorrt_llm::runtime::DecodingInput::finishReasons__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a7bcfc1dcf4652972d5d9b9e2926614c5"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finishReasons</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13finishReasonsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Optional parameters Finished states at current iteration (skip decoding step of a request if true), [batchSize, beamWidth] on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput19sequenceLimitLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput19sequenceLimitLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput19sequenceLimitLengthE"></span><span id="tensorrt_llm::runtime::DecodingInput::sequenceLimitLength__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ae6e1f98f774d7800fb5e8c18bf08a74f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">sequenceLimitLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput19sequenceLimitLengthE" title="Link to this definition">#</a><br /></dt>
<dd><p>The maximum sequence length for each sequence in the batch, [batchSize] on gpu. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13embeddingBiasE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13embeddingBiasE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13embeddingBiasE"></span><span id="tensorrt_llm::runtime::DecodingInput::embeddingBias__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ac7ae35915523b0cae76b9a628d2f8755"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">embeddingBias</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13embeddingBiasE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput7lengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput7lengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput7lengthsE"></span><span id="tensorrt_llm::runtime::DecodingInput::lengths__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a6adc71efd0aa8cc7bc3430204b4e71a8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput7lengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13badWordsListsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13badWordsListsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13badWordsListsE"></span><span id="tensorrt_llm::runtime::DecodingInput::badWordsLists__std::vector:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a88c9c583c32c2f1c3b36f7f426a5b369"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">badWordsLists</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13badWordsListsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsPtrsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12badWordsPtrsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12badWordsPtrsE"></span><span id="tensorrt_llm::runtime::DecodingInput::badWordsPtrs__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1afa6cbf9f8703ccf8bfedd7f24358cdd4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">badWordsPtrs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsPtrsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsLensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12badWordsLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12badWordsLensE"></span><span id="tensorrt_llm::runtime::DecodingInput::badWordsLens__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a1c15b3be4546d48f3e508abaf8f5afce"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">badWordsLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsLensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput14stopWordsListsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput14stopWordsListsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput14stopWordsListsE"></span><span id="tensorrt_llm::runtime::DecodingInput::stopWordsLists__std::vector:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a7badda2a04bb863a12c0d22381844c44"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stopWordsLists</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14stopWordsListsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsPtrsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13stopWordsPtrsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13stopWordsPtrsE"></span><span id="tensorrt_llm::runtime::DecodingInput::stopWordsPtrs__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a1a912305ebcd3788cc484b51ae6d97d7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stopWordsPtrs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsPtrsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsLensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput13stopWordsLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput13stopWordsLensE"></span><span id="tensorrt_llm::runtime::DecodingInput::stopWordsLens__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a830d90aeb7e6facbb8195d8cca055ccb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">stopWordsLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsLensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput17noRepeatNgramSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput17noRepeatNgramSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput17noRepeatNgramSizeE"></span><span id="tensorrt_llm::runtime::DecodingInput::noRepeatNgramSize__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ae80128aeb288e4aab05278ca2e2512bd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">noRepeatNgramSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput17noRepeatNgramSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput16cacheIndirectionE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput16cacheIndirectionE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput16cacheIndirectionE"></span><span id="tensorrt_llm::runtime::DecodingInput::cacheIndirection__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a6eccbda69f8c35e1eda57e9eb24ca930"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheIndirection</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput16cacheIndirectionE" title="Link to this definition">#</a><br /></dt>
<dd><p>Parameters for beam search KV cache index for beam search, [batchSize, beamWidth, maxSeqLen] on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15generationStepsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15generationStepsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15generationStepsE"></span><span id="tensorrt_llm::runtime::DecodingInput::generationSteps__std::optional:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ad894d851422bf946b3a61963f4b8a4c4"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationSteps</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15generationStepsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Steps of each request, for Variable-Beam-Width-Search, [batchSize]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12medusaInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12medusaInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12medusaInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::medusaInputs__std::optional:MedusaInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ad5cf2f6414e990319fa0ffaf3a95203a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE" title="tensorrt_llm::runtime::DecodingInput::MedusaInputs"><span class="n"><span class="pre">MedusaInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12medusaInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25explicitDraftTokensInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25explicitDraftTokensInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25explicitDraftTokensInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::explicitDraftTokensInputs__std::optional:ExplicitDraftTokensInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a7bd2778e96e29dafa69f792309aa6046"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE" title="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs"><span class="n"><span class="pre">ExplicitDraftTokensInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">explicitDraftTokensInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25explicitDraftTokensInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15lookaheadInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15lookaheadInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15lookaheadInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::lookaheadInputs__std::optional:LookaheadInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1ad6524d087f1dfe1873c4f75d4007d9f9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE" title="tensorrt_llm::runtime::DecodingInput::LookaheadInputs"><span class="n"><span class="pre">LookaheadInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lookaheadInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15lookaheadInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25externalDraftTokensInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25externalDraftTokensInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25externalDraftTokensInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::externalDraftTokensInputs__std::optional:ExternalDraftTokensInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1a2800c0b814e6287f14f1d52098e66282"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE" title="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs"><span class="n"><span class="pre">ExternalDraftTokensInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">externalDraftTokensInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25externalDraftTokensInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11eagleInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11eagleInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11eagleInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::eagleInputs__std::optional:EagleInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1affb7d2c439ced1b7adcf8d00aca41947"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputsE" title="tensorrt_llm::runtime::DecodingInput::EagleInputs"><span class="n"><span class="pre">EagleInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11eagleInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs11EagleInputsE14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs11EagleInputsE14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs11EagleInputsE14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::EagleInputs__TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr.TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a097428c4d295942c0117cd8061ceccf4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nextDraftTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nextDraftLens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">nextDraftPaths</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">lastDraftTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">lastDraftLens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">lastDraftPaths</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">acceptedTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">acceptedLens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">acceptedPathIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">chunkedContextNextTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs11EagleInputsE14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15nextDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs15nextDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs15nextDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::nextDraftTokens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a9d5c2128f6987a534f7857069f4df44d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15nextDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13nextDraftLensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs13nextDraftLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs13nextDraftLensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::nextDraftLens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a965fb843227f57cb06d2ee45791b773d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13nextDraftLensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14nextDraftPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs14nextDraftPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs14nextDraftPathsE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::nextDraftPaths__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a7b767bcb8ac75ab7d401a576d6223ab3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14nextDraftPathsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15lastDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs15lastDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs15lastDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::lastDraftTokens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a9fa43bad08b57f2650886493c6523631"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15lastDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13lastDraftLensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs13lastDraftLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs13lastDraftLensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::lastDraftLens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a9250c2234d9562e1a2c727c04897c9c7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastDraftLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13lastDraftLensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14lastDraftPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs14lastDraftPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs14lastDraftPathsE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::lastDraftPaths__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a93c600e33b2ccae8b113771bb724869c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastDraftPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14lastDraftPathsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14acceptedTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs14acceptedTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs14acceptedTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::acceptedTokens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a84333d7cd56cb5dac547f2c277bbfd9e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14acceptedTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs12acceptedLensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs12acceptedLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs12acceptedLensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::acceptedLens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1aabb443557155e294c75d283b5ca5776f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs12acceptedLensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15acceptedPathIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs15acceptedPathIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs15acceptedPathIdsE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::acceptedPathIds__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a4047afccdec273441c71aff5f259d5bf"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedPathIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15acceptedPathIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs24chunkedContextNextTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs24chunkedContextNextTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs24chunkedContextNextTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::chunkedContextNextTokens__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a7fd68f13a66f6f2e8c814a7700fb9056"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">chunkedContextNextTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs24chunkedContextNextTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs8seqSlotsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput11EagleInputs8seqSlotsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput11EagleInputs8seqSlotsE"></span><span id="tensorrt_llm::runtime::DecodingInput::EagleInputs::seqSlots__TensorConstPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1EagleInputs_1a2fb8510fab43cee499a8179534fab3d2"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">seqSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs8seqSlotsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExplicitDraftTokensInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15nextDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15nextDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15nextDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::nextDraftTokens__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1ab33eb98ffb56f34db936916707a02658"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15nextDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextFlatTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextFlatTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextFlatTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::nextFlatTokens__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a7ad74fb97965e08bb1a73cd19a45d14b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextFlatTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextFlatTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16nextDraftIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16nextDraftIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16nextDraftIndicesE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::nextDraftIndices__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1accc9adb18c4d965102d87fa2b630b277"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16nextDraftIndicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextDraftProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextDraftProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextDraftProbsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::nextDraftProbs__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a831ac5a03a692eaf6e34cb447e5e8301"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextDraftProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15lastDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15lastDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15lastDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::lastDraftTokens__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a9536a95353e9da425f7d3239765a7ac8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15lastDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16lastDraftIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16lastDraftIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16lastDraftIndicesE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::lastDraftIndices__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a9b35756e07900384197581a3b91aeb62"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastDraftIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16lastDraftIndicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs5masksE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs5masksE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs5masksE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::masks__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a1a02a579fcf08853b3c115771935e568"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">masks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs5masksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs17packedPositionIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs17packedPositionIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs17packedPositionIdsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::packedPositionIds__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a916530fdfd03ed41753e183c068f2754"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedPositionIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs17packedPositionIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathLengthsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::bestPathLengths__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a14ee72c4ff8a12bddb2cc0e2145ef127"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bestPathLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathIndicesE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::bestPathIndices__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1ad21ef1fb71e9f00a6f67a95086b38deb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bestPathIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathIndicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21nextGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21nextGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21nextGenerationLengthsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::nextGenerationLengths__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a974f61b17e2232c378d2939b08c7507d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21nextGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs19lastPositionIdsBaseE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs19lastPositionIdsBaseE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs19lastPositionIdsBaseE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::lastPositionIdsBase__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1a50002e1cd1d62e167745cc694fefb451"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastPositionIdsBase</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs19lastPositionIdsBaseE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21lastGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21lastGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21lastGenerationLengthsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::lastGenerationLengths__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1ad10825bb06f20d4ce8f67a630c0e04e4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lastGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21lastGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs18maxGenLengthDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs18maxGenLengthDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs18maxGenLengthDeviceE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::maxGenLengthDevice__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1ac1fd1f48b7668c9f9f295c9ad8ccfad0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxGenLengthDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs18maxGenLengthDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs8seqSlotsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs8seqSlotsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs8seqSlotsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs::seqSlots__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExplicitDraftTokensInputs_1abfd632f47aff831cd43f55eb40b1ea82"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">seqSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs8seqSlotsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExternalDraftTokensInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11draftLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11draftLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11draftLogitsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::draftLogits__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1addf6ccc0d11838a16a36d4d574f2149b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11draftLogitsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs10draftProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs10draftProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs10draftProbsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::draftProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1ab39186249926f9f16fc399f1f47db321"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs10draftProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11targetProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11targetProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11targetProbsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::targetProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a1257278db85997fab8bc0c6d604e95c1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">targetProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11targetProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14numDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14numDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14numDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::numDraftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a0d5d346e1f80a3bac75fcab5b7c58369"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14numDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18numDraftTokensHostE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18numDraftTokensHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18numDraftTokensHostE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::numDraftTokensHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a5e1a6471a16f836b972e640061419f4f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numDraftTokensHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18numDraftTokensHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs13draftTokenIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs13draftTokenIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs13draftTokenIdsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::draftTokenIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a8ff083621c48c2ef0a74847f17925c6a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftTokenIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs13draftTokenIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14useDraftLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14useDraftLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14useDraftLogitsE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::useDraftLogits__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1ae15a164a475300b1e268048fa080c00e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useDraftLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14useDraftLogitsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18useDraftLogitsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18useDraftLogitsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18useDraftLogitsHostE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::useDraftLogitsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a5223b07547875357d19c7e9bf2bf0c9b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useDraftLogitsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18useDraftLogitsHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs4stepE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs4stepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs4stepE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::step__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1a7d222ffc1f7db7497ce4a809bcf3779a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">step</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs4stepE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs17constantThresholdE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs17constantThresholdE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs17constantThresholdE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::constantThreshold__float"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1ad6952502823a4452f686bc2c5a574f5b"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">constantThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs17constantThresholdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs28useRandomAcceptanceThresholdE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs28useRandomAcceptanceThresholdE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs28useRandomAcceptanceThresholdE"></span><span id="tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs::useRandomAcceptanceThreshold__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1ExternalDraftTokensInputs_1ace748e3667f3462d8edb615c808a78d7"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useRandomAcceptanceThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs28useRandomAcceptanceThresholdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::LookaheadInputs"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1LookaheadInputs"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputs13tokensPerStepE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput15LookaheadInputs13tokensPerStepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput15LookaheadInputs13tokensPerStepE"></span><span id="tensorrt_llm::runtime::DecodingInput::LookaheadInputs::tokensPerStep__TensorPtr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DecodingInput_1_1LookaheadInputs_1a646f2bcd543ac02e92b3c8cc40e2c920"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokensPerStep</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputs13tokensPerStepE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs11medusaPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputs11medusaPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputs11medusaPathsE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs::medusaPaths__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs_1a5307b7803b035e1ce0814dc8523a8f60"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs11medusaPathsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxTokensPerStep, maxMedusaHeads + 1], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs13medusaTreeIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputs13medusaTreeIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputs13medusaTreeIdsE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs::medusaTreeIds__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs_1ad5ce6d5babb4b1a5c60c1203e9594f87"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaTreeIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs13medusaTreeIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxTokensPerStep], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs12medusaLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputs12medusaLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputs12medusaLogitsE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs::medusaLogits__std::vector:std::vector:TensorPtr::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs_1a61552ddc3f54658c465929a297912036"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs12medusaLogitsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize][maxAcceptedDraftTokensPerStep][maxDraftTokens + 1, vocabSizePadded], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs22medusaCurTokensPerStepE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputs22medusaCurTokensPerStepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputs22medusaCurTokensPerStepE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs::medusaCurTokensPerStep__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs_1a1a8c2fcc38cb3c8c3c62ba80bc0cf60f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaCurTokensPerStep</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs22medusaCurTokensPerStepE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs25medusaTargetTokensPerStepE">
<span id="_CPPv3N12tensorrt_llm7runtime13DecodingInput12MedusaInputs25medusaTargetTokensPerStepE"></span><span id="_CPPv2N12tensorrt_llm7runtime13DecodingInput12MedusaInputs25medusaTargetTokensPerStepE"></span><span id="tensorrt_llm::runtime::DecodingInput::MedusaInputs::medusaTargetTokensPerStep__TensorConstPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingInput_1_1MedusaInputs_1a65af55c74760e3aef0f03177f604e849"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE" title="tensorrt_llm::runtime::DecodingInput::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">medusaTargetTokensPerStep</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs25medusaTargetTokensPerStepE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize], on gpu </p>
</dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="gptjsonconfig-h">
<h2>gptJsonConfig.h<a class="headerlink" href="#gptjsonconfig-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfigE"></span><span id="tensorrt_llm::runtime::GptJsonConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GptJsonConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig13GptJsonConfigENSt6stringENSt6stringENSt6stringE10SizeType3210SizeType3210SizeType3210SizeType3211ModelConfigNSt8optionalI15RuntimeDefaultsEE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig13GptJsonConfigENSt6stringENSt6stringENSt6stringE10SizeType3210SizeType3210SizeType3210SizeType3211ModelConfigNSt8optionalI15RuntimeDefaultsEE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig13GptJsonConfigENSt6stringENSt6stringENSt6stringE10SizeType3210SizeType3210SizeType3210SizeType3211ModelConfigNSt8optionalI15RuntimeDefaultsEE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::GptJsonConfig__ss.ss.ss.SizeType32.SizeType32.SizeType32.SizeType32.ModelConfig.std::optional:RuntimeDefaults:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a712400cdaee8fc97ce35e2299ab1f4af"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GptJsonConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">version</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">precision</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensorParallelism</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">pipelineParallelism</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">contextParallelism</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gpusPerNode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="tensorrt_llm::runtime::RuntimeDefaults"><span class="n"><span class="pre">RuntimeDefaults</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">runtimeDefaults</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig13GptJsonConfigENSt6stringENSt6stringENSt6stringE10SizeType3210SizeType3210SizeType3210SizeType3211ModelConfigNSt8optionalI15RuntimeDefaultsEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getModelConfigEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig14getModelConfigEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig14getModelConfigEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getModelConfigC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a5c3285f2c061c1330ca11e1343cf89a2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getModelConfig</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getModelConfigEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig21getModelConfigMutableEv">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig21getModelConfigMutableEv"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig21getModelConfigMutableEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getModelConfigMutable"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ad6d1468e250b06a17e67e10b7d94d823"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getModelConfigMutable</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig21getModelConfigMutableEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig7getNameEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig7getNameEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig7getNameEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getNameC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a5270fbe10703e1e5fe5d52d104b30cfb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig7getNameEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig10getVersionEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig10getVersionEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig10getVersionEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getVersionC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a5900e7d368979420ec10c7635e656ea3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getVersion</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig10getVersionEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getPrecisionEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig12getPrecisionEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig12getPrecisionEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getPrecisionC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a353424dd7b526c63e5d03fedc5314ed1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getPrecision</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getPrecisionEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig20getTensorParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig20getTensorParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig20getTensorParallelismEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getTensorParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1aac7f508e90ae3316d1065b0eb68f8aee"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getTensorParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig20getTensorParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig22getPipelineParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig22getPipelineParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig22getPipelineParallelismEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getPipelineParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a075d679a1b8c80bc303441308a9c28bc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPipelineParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig22getPipelineParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig21getContextParallelismEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig21getContextParallelismEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig21getContextParallelismEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getContextParallelismCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ad441fbfc413a64d424510520526a38b7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getContextParallelism</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig21getContextParallelismEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getGpusPerNodeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig14getGpusPerNodeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig14getGpusPerNodeEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getGpusPerNodeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ad3b5423d2e378a551e73abce93609667"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getGpusPerNode</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getGpusPerNodeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getWorldSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig12getWorldSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig12getWorldSizeEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getWorldSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a3023e339b22eca5cf4ba14304c46b567"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getWorldSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getWorldSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig18getRuntimeDefaultsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig18getRuntimeDefaultsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig18getRuntimeDefaultsEv"></span><span id="tensorrt_llm::runtime::GptJsonConfig::getRuntimeDefaultsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ac8e7e4aa82a1c14146563b5ee7acd7b8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="tensorrt_llm::runtime::RuntimeDefaults"><span class="n"><span class="pre">RuntimeDefaults</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getRuntimeDefaults</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig18getRuntimeDefaultsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfigRKNSt6stringE">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfigRKNSt6stringE"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfigRKNSt6stringE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::engineFilename__WorldConfigCR.ssCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ac663861699ef7f573f64d5fdb89f14af"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineFilename</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">model</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfigRKNSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfig">
<span id="_CPPv3NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfig"></span><span id="_CPPv2NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfig"></span><span id="tensorrt_llm::runtime::GptJsonConfig::engineFilename__WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a9ce08952a809fbe5859a685215dd7258"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineFilename</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt6stringE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt6stringE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt6stringE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::parse__ssCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a258c808e0e2c5445c807ae2f0a257782"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE" title="tensorrt_llm::runtime::GptJsonConfig"><span class="n"><span class="pre">GptJsonConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">parse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">json</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt6stringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERNSt7istreamE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig5parseERNSt7istreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig5parseERNSt7istreamE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::parse__isR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1af74ea4db9854b1f0ddd8aaa0d4a48f39"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE" title="tensorrt_llm::runtime::GptJsonConfig"><span class="n"><span class="pre">GptJsonConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">parse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">istream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">json</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERNSt7istreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt10filesystem4pathE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt10filesystem4pathE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt10filesystem4pathE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::parse__std::filesystem::pathCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a959c20c852124ced898b12bbdeb5d79f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE" title="tensorrt_llm::runtime::GptJsonConfig"><span class="n"><span class="pre">GptJsonConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">parse</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">path</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt10filesystem4pathE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5mNameE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig5mNameE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig5mNameE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mName__ssC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1aa23cc7f0c9ad465ffdb3c1950876b470"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mName</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5mNameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig8mVersionE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig8mVersionE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig8mVersionE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mVersion__ssC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a254b88b5fe490e3835177dd32ecb2c8f"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mVersion</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig8mVersionE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig10mPrecisionE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig10mPrecisionE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig10mPrecisionE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mPrecision__ssC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1acf01c9bba53cd18d0ece1026ef19e485"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPrecision</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig10mPrecisionE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig18mTensorParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig18mTensorParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig18mTensorParallelismE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mTensorParallelism__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ae1cc00036f7255a4d9580f833f8ab146"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mTensorParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig18mTensorParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig20mPipelineParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig20mPipelineParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig20mPipelineParallelismE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mPipelineParallelism__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a5eef99e58f6d89bb63040884a26f4dd6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPipelineParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig20mPipelineParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig19mContextParallelismE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig19mContextParallelismE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig19mContextParallelismE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mContextParallelism__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a401239ff678e6761f2ab44f2da2e8480"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mContextParallelism</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig19mContextParallelismE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mGpusPerNodeE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig12mGpusPerNodeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig12mGpusPerNodeE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mGpusPerNode__SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1a719eb6ee79619bdf2997de06d7d93d52"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mGpusPerNode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mGpusPerNodeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mModelConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig12mModelConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig12mModelConfigE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mModelConfig__ModelConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1ada6b440f8e35d7cc663ae5aee2202655"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mModelConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mModelConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13GptJsonConfig16mRuntimeDefaultsE">
<span id="_CPPv3N12tensorrt_llm7runtime13GptJsonConfig16mRuntimeDefaultsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13GptJsonConfig16mRuntimeDefaultsE"></span><span id="tensorrt_llm::runtime::GptJsonConfig::mRuntimeDefaults__std::optional:RuntimeDefaults:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptJsonConfig_1abb7f59b5946ca00c70649e7c2554028a"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE" title="tensorrt_llm::runtime::RuntimeDefaults"><span class="n"><span class="pre">RuntimeDefaults</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRuntimeDefaults</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig16mRuntimeDefaultsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="ipcnvlsmemory-h">
<h2>ipcNvlsMemory.h<a class="headerlink" href="#ipcnvlsmemory-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17MPI_group_barrierENSt3setIiEE">
<span id="_CPPv3N12tensorrt_llm7runtime17MPI_group_barrierENSt3setIiEE"></span><span id="_CPPv2N12tensorrt_llm7runtime17MPI_group_barrierENSt3setIiEE"></span><span id="tensorrt_llm::runtime::MPI_group_barrier__std::set:i:"></span><span class="target" id="ipcNvlsMemory_8h_1aebf924b03acf459ef92744d835236ef4"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MPI_group_barrier</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">set</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ranks</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17MPI_group_barrierENSt3setIiEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16ipcNvlsSupportedEv">
<span id="_CPPv3N12tensorrt_llm7runtime16ipcNvlsSupportedEv"></span><span id="_CPPv2N12tensorrt_llm7runtime16ipcNvlsSupportedEv"></span><span id="tensorrt_llm::runtime::ipcNvlsSupported"></span><span class="target" id="ipcNvlsMemory_8h_1ac0edf48a562b014734765ea635b464e9"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipcNvlsSupported</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16ipcNvlsSupportedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15ipcNvlsAllocateE6size_tNSt3setIiEE">
<span id="_CPPv3N12tensorrt_llm7runtime15ipcNvlsAllocateE6size_tNSt3setIiEE"></span><span id="_CPPv2N12tensorrt_llm7runtime15ipcNvlsAllocateE6size_tNSt3setIiEE"></span><span id="tensorrt_llm::runtime::ipcNvlsAllocate__s.std::set:i:"></span><span class="target" id="ipcNvlsMemory_8h_1a99e9646f8d172a6c37514a29897217b7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE" title="tensorrt_llm::runtime::IpcNvlsHandle"><span class="n"><span class="pre">IpcNvlsHandle</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">ipcNvlsAllocate</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">set</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ranks</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15ipcNvlsAllocateE6size_tNSt3setIiEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11ipcNvlsFreeEP13IpcNvlsHandle">
<span id="_CPPv3N12tensorrt_llm7runtime11ipcNvlsFreeEP13IpcNvlsHandle"></span><span id="_CPPv2N12tensorrt_llm7runtime11ipcNvlsFreeEP13IpcNvlsHandle"></span><span id="tensorrt_llm::runtime::ipcNvlsFree__IpcNvlsHandleP"></span><span class="target" id="ipcNvlsMemory_8h_1a469959df8a9c2648a259c9f67f923faf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipcNvlsFree</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE" title="tensorrt_llm::runtime::IpcNvlsHandle"><span class="n"><span class="pre">IpcNvlsHandle</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">handle</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11ipcNvlsFreeEP13IpcNvlsHandle" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DeviceAllocationNvls</span></span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls20DeviceAllocationNvlsEv">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls20DeviceAllocationNvlsEv"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls20DeviceAllocationNvlsEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::DeviceAllocationNvls"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a404f3311794bc7f6d9db6e2d77035615"></span><span class="sig-name descname"><span class="n"><span class="pre">DeviceAllocationNvls</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls20DeviceAllocationNvlsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvlsD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvlsD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvlsD0Ev"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::~DeviceAllocationNvls"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a168236c20f3b0422b2d4e8b8ae6955b4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~DeviceAllocationNvls</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvlsD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls5resetE6size_tNSt3setIiEE">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls5resetE6size_tNSt3setIiEE"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls5resetE6size_tNSt3setIiEE"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::reset__s.std::set:i:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a035c79859bef3d55f303ec2e8e71a6d1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reset</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">set</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ranks</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls5resetE6size_tNSt3setIiEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls19getMulticastPointerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime20DeviceAllocationNvls19getMulticastPointerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime20DeviceAllocationNvls19getMulticastPointerEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::getMulticastPointerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a41aa7ccd4b8fb509d02d6ffc91e57177"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE" title="tensorrt_llm::runtime::DeviceAllocationNvls::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getMulticastPointer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls19getMulticastPointerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls17getUnicastPointerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime20DeviceAllocationNvls17getUnicastPointerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime20DeviceAllocationNvls17getUnicastPointerEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::getUnicastPointerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1af9a8cb5d6bca9b7a1c2206dcd261b4c8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE" title="tensorrt_llm::runtime::DeviceAllocationNvls::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getUnicastPointer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls17getUnicastPointerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls21getIpcUnicastPointersEv">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls21getIpcUnicastPointersEv"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls21getIpcUnicastPointersEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::getIpcUnicastPointers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1aa55c9818e3c7d8f82667e26c68077d53"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE" title="tensorrt_llm::runtime::DeviceAllocationNvls::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getIpcUnicastPointers</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls21getIpcUnicastPointersEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls11getCapacityEv">
<span id="_CPPv3NK12tensorrt_llm7runtime20DeviceAllocationNvls11getCapacityEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime20DeviceAllocationNvls11getCapacityEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::getCapacityC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1af47582b908d5d5a051aeb6d9236993eb"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCapacity</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls11getCapacityEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls4freeEv">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls4freeEv"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls4freeEv"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::free"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a6353ceabe8a5ec6ea462a7099710d0f4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">free</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls4freeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls9_capacityE">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls9_capacityE"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls9_capacityE"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::_capacity__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1aa45de6d0e763547746e9e905fe5f1a95"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">_capacity</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls9_capacityE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls7_handleE">
<span id="_CPPv3N12tensorrt_llm7runtime20DeviceAllocationNvls7_handleE"></span><span id="_CPPv2N12tensorrt_llm7runtime20DeviceAllocationNvls7_handleE"></span><span id="tensorrt_llm::runtime::DeviceAllocationNvls::_handle__IpcNvlsHandleP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DeviceAllocationNvls_1a8d5fb46a384c1c3d8753f45cfb9f48d7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE" title="tensorrt_llm::runtime::IpcNvlsHandle"><span class="n"><span class="pre">IpcNvlsHandle</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">_handle</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls7_handleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandleE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandleE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IpcNvlsHandle</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle4sizeE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::size__s"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1aa1aa3b56d1839944112c8cf568b9e1e7"></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6uc_ptrE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle6uc_ptrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle6uc_ptrE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::uc_ptr__uintptr_t"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1a50fd446b0b4016af75aaa695287872ae"></span><span class="n"><span class="pre">uintptr_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">uc_ptr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6uc_ptrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6mc_ptrE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle6mc_ptrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle6mc_ptrE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::mc_ptr__uintptr_t"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1ac3b719f71904e2a162d8d504323423ce"></span><span class="n"><span class="pre">uintptr_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mc_ptr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6mc_ptrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle11ipc_uc_ptrsE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle11ipc_uc_ptrsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle11ipc_uc_ptrsE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::ipc_uc_ptrs__std::vector:uintptr_t:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1a0ea4380f35e42751041723bfe4ce6a2e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">uintptr_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipc_uc_ptrs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle11ipc_uc_ptrsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5uc_vaE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle5uc_vaE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle5uc_vaE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::uc_va__CUdeviceptr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1ab25cfb4149083956c3f51bb54aa27cee"></span><span class="n"><span class="pre">CUdeviceptr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">uc_va</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5uc_vaE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5mc_vaE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle5mc_vaE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle5mc_vaE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::mc_va__CUdeviceptr"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1abe378c4d14ce45df70a4c7bc2686edb6"></span><span class="n"><span class="pre">CUdeviceptr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mc_va</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5mc_vaE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle10ipc_uc_vasE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle10ipc_uc_vasE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle10ipc_uc_vasE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::ipc_uc_vas__std::vector:CUdeviceptr:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1aeb26c3a96799a3bc0b76f0102033f701"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">CUdeviceptr</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipc_uc_vas</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle10ipc_uc_vasE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9uc_handleE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle9uc_handleE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle9uc_handleE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::uc_handle__CUmemGenericAllocationHandle"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1a3e48a5ab7befee9f757dd5350aa48f89"></span><span class="n"><span class="pre">CUmemGenericAllocationHandle</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">uc_handle</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9uc_handleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9mc_handleE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle9mc_handleE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle9mc_handleE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::mc_handle__CUmemGenericAllocationHandle"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1aced659921dd2470ec4d30754837941f8"></span><span class="n"><span class="pre">CUmemGenericAllocationHandle</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mc_handle</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9mc_handleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle14ipc_uc_handlesE">
<span id="_CPPv3N12tensorrt_llm7runtime13IpcNvlsHandle14ipc_uc_handlesE"></span><span id="_CPPv2N12tensorrt_llm7runtime13IpcNvlsHandle14ipc_uc_handlesE"></span><span id="tensorrt_llm::runtime::IpcNvlsHandle::ipc_uc_handles__std::vector:CUmemGenericAllocationHandle:"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1IpcNvlsHandle_1a39a66830f800f7757206bd7d6e1c27bc"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">CUmemGenericAllocationHandle</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ipc_uc_handles</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle14ipc_uc_handlesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="samplingconfig-h">
<h2>samplingConfig.h<a class="headerlink" href="#samplingconfig-h" title="Link to this heading">#</a></h2>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-defines">Defines</p>
<dl class="cpp macro">
<dt class="sig sig-object cpp" id="c.SET_FROM_OPTIONAL">
<span class="target" id="samplingConfig_8h_1ae2a9ee9b68dd5ae5302af8d853bab8da"></span><span class="sig-name descname"><span class="n"><span class="pre">SET_FROM_OPTIONAL</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">varName</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">VarName</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">VarType</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#c.SET_FROM_OPTIONAL" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfigE"></span><span id="tensorrt_llm::runtime::SamplingConfig"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigE10SizeType32"></span><span id="tensorrt_llm::runtime::SamplingConfig::SamplingConfig__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a21ca969b785842a734cb5f972e68706d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE"></span><span id="tensorrt_llm::runtime::SamplingConfig::SamplingConfig__std::vector:SamplingConfig:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a56ba4c7f8091d42e62e8651078be6f18"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE" title="tensorrt_llm::runtime::SamplingConfig::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">configs</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKN8executor14SamplingConfigERKNSt8optionalIN8executor25ExternalDraftTokensConfigEEE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKN8executor14SamplingConfigERKNSt8optionalIN8executor25ExternalDraftTokensConfigEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKN8executor14SamplingConfigERKNSt8optionalIN8executor25ExternalDraftTokensConfigEEE"></span><span id="tensorrt_llm::runtime::SamplingConfig::SamplingConfig__executor::SamplingConfigCR.std::optional:executor::ExternalDraftTokensConfig:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a3344128d4ca3c4a11fe8680f1fc240dc"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SamplingConfig</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor14SamplingConfigE" title="tensorrt_llm::executor::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor25ExternalDraftTokensConfigE" title="tensorrt_llm::executor::ExternalDraftTokensConfig"><span class="n"><span class="pre">ExternalDraftTokensConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">externalDraftTokensConfig</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKN8executor14SamplingConfigERKNSt8optionalIN8executor25ExternalDraftTokensConfigEEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig8validateEv">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig8validateEv"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig8validateEv"></span><span id="tensorrt_llm::runtime::SamplingConfig::validate"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a1ee5ddc2543fe7b3dc6389b7d8fc763b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">validate</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig8validateEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesEbRK6OptVecI1TE1T">
<span id="_CPPv3I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesERK6OptVecI1TE1T"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesERK6OptVecI1TE1T"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a03ce780bf1e9b1c48793e6b8dff319bd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useDefaultValues</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesEbRK6OptVecI1TE1T" title="tensorrt_llm::runtime::SamplingConfig::useDefaultValues::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">vec</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesEbRK6OptVecI1TE1T" title="tensorrt_llm::runtime::SamplingConfig::useDefaultValues::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">defaultValue</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesEbRK6OptVecI1TE1T" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14SamplingConfigeqERK14SamplingConfig">
<span id="_CPPv3NK12tensorrt_llm7runtime14SamplingConfigeqERK14SamplingConfig"></span><span id="_CPPv2NK12tensorrt_llm7runtime14SamplingConfigeqERK14SamplingConfig"></span><span id="tensorrt_llm::runtime::SamplingConfig::eq-operator__SamplingConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a7d909d5f82efa13555105b8373cb1451"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfigeqERK14SamplingConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14SamplingConfig17getNumReturnBeamsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14SamplingConfig17getNumReturnBeamsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14SamplingConfig17getNumReturnBeamsEv"></span><span id="tensorrt_llm::runtime::SamplingConfig::getNumReturnBeamsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a5abd37189f9136af94760c5580f249e7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumReturnBeams</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfig17getNumReturnBeamsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14SamplingConfig15getMaxBeamWidthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14SamplingConfig15getMaxBeamWidthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14SamplingConfig15getMaxBeamWidthEv"></span><span id="tensorrt_llm::runtime::SamplingConfig::getMaxBeamWidthC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ab711cce841f1af53a726e6e4131f6d16"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxBeamWidth</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfig15getMaxBeamWidthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig9beamWidthE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig9beamWidthE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig9beamWidthE"></span><span id="tensorrt_llm::runtime::SamplingConfig::beamWidth__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a52bb6d11985ba57e1227da7d1ecc0fd0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">beamWidth</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9beamWidthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig18numReturnSequencesE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig18numReturnSequencesE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig18numReturnSequencesE"></span><span id="tensorrt_llm::runtime::SamplingConfig::numReturnSequences__std::optional:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ae761961bb83f67c77d94b56125d298a6"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numReturnSequences</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig18numReturnSequencesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig11temperatureE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig11temperatureE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig11temperatureE"></span><span id="tensorrt_llm::runtime::SamplingConfig::temperature__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1aa0234e25caafb8f7e2540e635354f1a7"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">temperature</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig11temperatureE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig19originalTemperatureE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig19originalTemperatureE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig19originalTemperatureE"></span><span id="tensorrt_llm::runtime::SamplingConfig::originalTemperature__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1aaa22b7ffef3a4e0e15efb990fe8907b2"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">originalTemperature</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig19originalTemperatureE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig9minLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig9minLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig9minLengthE"></span><span id="tensorrt_llm::runtime::SamplingConfig::minLength__OptVec:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a30d0f89f7035ccd82315aa0bc3fd7182"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">minLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9minLengthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig17repetitionPenaltyE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig17repetitionPenaltyE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig17repetitionPenaltyE"></span><span id="tensorrt_llm::runtime::SamplingConfig::repetitionPenalty__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ab267d53af1ff989a2cda73069dfd42fa"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">repetitionPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17repetitionPenaltyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig15presencePenaltyE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig15presencePenaltyE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig15presencePenaltyE"></span><span id="tensorrt_llm::runtime::SamplingConfig::presencePenalty__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1af40f62527a6a93da70def3daafdc8001"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">presencePenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig15presencePenaltyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig16frequencyPenaltyE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig16frequencyPenaltyE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig16frequencyPenaltyE"></span><span id="tensorrt_llm::runtime::SamplingConfig::frequencyPenalty__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a1d642826eaf790461c458263f2ad90c5"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">frequencyPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig16frequencyPenaltyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig17noRepeatNgramSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig17noRepeatNgramSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig17noRepeatNgramSizeE"></span><span id="tensorrt_llm::runtime::SamplingConfig::noRepeatNgramSize__OptVec:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1aeedb42ce71155887fc3c9aea45a423ce"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">noRepeatNgramSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17noRepeatNgramSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig14outputLogProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig14outputLogProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig14outputLogProbsE"></span><span id="tensorrt_llm::runtime::SamplingConfig::outputLogProbs__OptVec:b:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ad55655684229d974d259c5222ad613f7"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outputLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14outputLogProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig11cumLogProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig11cumLogProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig11cumLogProbsE"></span><span id="tensorrt_llm::runtime::SamplingConfig::cumLogProbs__OptVec:b:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a19bc5c481edee164410a04bacbbe81ed"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig11cumLogProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topKE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig4topKE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig4topKE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topK__OptVec:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a89d73ce271859ae8623309e97ef38a99"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topK</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topKE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topPE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig4topPE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig4topPE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topP__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a6d960ae9fb60fa44c616cf4b16a6977d"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topPE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig10randomSeedE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig10randomSeedE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig10randomSeedE"></span><span id="tensorrt_llm::runtime::SamplingConfig::randomSeed__OptVec:uint64_t:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ace2fdcbe3634f654db68096f7d89a494"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">uint64_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">randomSeed</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig10randomSeedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig9topPDecayE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig9topPDecayE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig9topPDecayE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topPDecay__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ad25e0aec9100bf05d100885677dfaa03"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topPDecay</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9topPDecayE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig7topPMinE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig7topPMinE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig7topPMinE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topPMin__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a2f772d5c0be4d9fe80fdf0e142a731d5"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topPMin</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig7topPMinE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig12topPResetIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig12topPResetIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig12topPResetIdsE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topPResetIds__OptVec:TokenIdType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1afffb3130864d729191225855b3144d94"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE" title="tensorrt_llm::runtime::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topPResetIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig12topPResetIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig4minPE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig4minPE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig4minPE"></span><span id="tensorrt_llm::runtime::SamplingConfig::minP__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a564ef0358d770060b6df52054d03cdfe"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">minP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4minPE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig23beamSearchDiversityRateE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig23beamSearchDiversityRateE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig23beamSearchDiversityRateE"></span><span id="tensorrt_llm::runtime::SamplingConfig::beamSearchDiversityRate__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a58a44a42e3086649b6b510222b007ac6"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">beamSearchDiversityRate</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig23beamSearchDiversityRateE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig13lengthPenaltyE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig13lengthPenaltyE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig13lengthPenaltyE"></span><span id="tensorrt_llm::runtime::SamplingConfig::lengthPenalty__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a393378cbfb39e5a147b88a8601050947"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lengthPenalty</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig13lengthPenaltyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig13earlyStoppingE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig13earlyStoppingE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig13earlyStoppingE"></span><span id="tensorrt_llm::runtime::SamplingConfig::earlyStopping__OptVec:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a947d2499aa3f805431f64206052dfdf3"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">earlyStopping</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig13earlyStoppingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig14beamWidthArrayE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig14beamWidthArrayE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig14beamWidthArrayE"></span><span id="tensorrt_llm::runtime::SamplingConfig::beamWidthArray__OptVec:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a703fd3e5fa163efd898e0e81107faa9a"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">beamWidthArray</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14beamWidthArrayE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig24draftAcceptanceThresholdE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig24draftAcceptanceThresholdE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig24draftAcceptanceThresholdE"></span><span id="tensorrt_llm::runtime::SamplingConfig::draftAcceptanceThreshold__OptVec:FloatType:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a914d673110a9a51924ec03567f2b6fb5"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="tensorrt_llm::runtime::SamplingConfig::FloatType"><span class="n"><span class="pre">FloatType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftAcceptanceThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig24draftAcceptanceThresholdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig15topKMedusaHeadsE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig15topKMedusaHeadsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig15topKMedusaHeadsE"></span><span id="tensorrt_llm::runtime::SamplingConfig::topKMedusaHeads__OptVec:std::vector:SizeType32::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1ad3bf39c4f2fd4e90a3af386b2f4b69f8"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">topKMedusaHeads</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig15topKMedusaHeadsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig17normalizeLogProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig17normalizeLogProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig17normalizeLogProbsE"></span><span id="tensorrt_llm::runtime::SamplingConfig::normalizeLogProbs__std::optional:b:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a7657ff197df68d0b7591497d9434983e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">normalizeLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17normalizeLogProbsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a78311bb676349c17d54fee63f3d54396"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FloatType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="kt"><span class="pre">float</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1a82ed7012baf2949351e80937329b530a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">OptVec</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecENSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecENSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1aa734148feaaa8708c45664ed0e293811"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">validateVec</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE" title="tensorrt_llm::runtime::SamplingConfig::validateVec::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">vec</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE" title="tensorrt_llm::runtime::SamplingConfig::validateVec::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">min</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE" title="tensorrt_llm::runtime::SamplingConfig::validateVec::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">max</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">nullopt</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-functions">Private Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T">
<span id="_CPPv3I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1SamplingConfig_1aaf40d6f66a7a24f1d0328ea68e104bef"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T" title="tensorrt_llm::runtime::SamplingConfig::fuseValues::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fuseValues</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">configs</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">function</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE" title="tensorrt_llm::runtime::SamplingConfig::OptVec"><span class="n"><span class="pre">OptVec</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T" title="tensorrt_llm::runtime::SamplingConfig::fuseValues::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">(</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n"><span class="pre">ci</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">accessor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T" title="tensorrt_llm::runtime::SamplingConfig::fuseValues::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">defaultValue</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="gptdecoderbatched-h">
<h2>gptDecoderBatched.h<a class="headerlink" href="#gptdecoderbatched-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm13batch_managerE">
<span id="_CPPv3N12tensorrt_llm13batch_managerE"></span><span id="_CPPv2N12tensorrt_llm13batch_managerE"></span><span id="tensorrt_llm::batch_manager"></span><span class="target" id="namespacetensorrt__llm_1_1batch__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batch_manager</span></span></span><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatchedE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatchedE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatchedE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GptDecoderBatched</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedE" title="tensorrt_llm::runtime::IGptDecoderBatched"><span class="n"><span class="pre">IGptDecoderBatched</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatchedE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;gptDecoderBatched.h&gt;</em></div>
<p>GPT decoder class with support for in-flight batching. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1aee79b622e8e07d57628f4482d32b340e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a1050f891cf2cd69288da22e97626d7be"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LlmRequestPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">LlmRequest</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a1a6731b1d1b083cacf268a341ef7d782"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestVector</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::LlmRequestPtr"><span class="n"><span class="pre">LlmRequestPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a71918575432e49931d0452cfb4c98a8d"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14SharedConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14SharedConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14SharedConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a2dd6a0a3bdccf9535c9df769033efe2e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SharedConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14SharedConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched17GptDecoderBatchedE13CudaStreamPtrRK23SpeculativeDecodingModeN8nvinfer18DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched17GptDecoderBatchedE13CudaStreamPtrRK23SpeculativeDecodingModeN8nvinfer18DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched17GptDecoderBatchedE13CudaStreamPtrRK23SpeculativeDecodingModeN8nvinfer18DataTypeE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::GptDecoderBatched__CudaStreamPtr.SpeculativeDecodingModeCR.nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1aca731fb55132a618ac1402370e6f55d6"></span><span class="sig-name descname"><span class="n"><span class="pre">GptDecoderBatched</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">speculativeDecodingMode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched17GptDecoderBatchedE13CudaStreamPtrRK23SpeculativeDecodingModeN8nvinfer18DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::setup__executor::DecodingModeCR.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.nvinfer1::DataType.ModelConfigCR.WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a4d3eccaa52123a48e1721d06f3e464a4"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setup</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindow</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokensPerStep</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup the decoder before calling <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a918a04e07f32ab5367f8b3e82697d7e5"><span class="std std-ref"><span class="pre">forward()</span></span></a></code></p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::disableLookahead__RequestVectorCR.TensorPtrCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a1e8c67d9db65b138260f721101ae83d7"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookahead</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE" title="tensorrt_llm::runtime::GptDecoderBatched::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">genRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd><p>Disable Lookahead decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::forwardAsync__decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1ad043b3f39e5b6efac043b9ed90ed1b7e"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardAsync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Run one step for all requests without blocking the host process and return the token for synchronization. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::forward__decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a918a04e07f32ab5367f8b3e82697d7e5"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forward</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Run one step for all requests and wait for completion on the host. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb">
<span id="_CPPv3NK12tensorrt_llm7runtime17GptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"></span><span id="_CPPv2NK12tensorrt_llm7runtime17GptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::finalize__decoder::DecoderStateCR.SizeType32.SamplingConfigCR.bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a555549b26a623b8e8bb212bf679a29a9"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finalize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="tensorrt_llm::runtime::decoder"><span class="n"><span class="pre">decoder</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="tensorrt_llm::runtime::decoder::DecoderState"><span class="n"><span class="pre">DecoderState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decoderState</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSlot</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">streaming</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb" title="Link to this definition">#</a><br /></dt>
<dd><p>Gather final beam search results for request <code class="docutils literal notranslate"><span class="pre">batchSlot</span></code>. Result will only be available after event returned. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::getDecoderState"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a5b62ea90ff74aa90cee081ce7b40a2ac"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="tensorrt_llm::runtime::decoder"><span class="n"><span class="pre">decoder</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="tensorrt_llm::runtime::decoder::DecoderState"><span class="n"><span class="pre">DecoderState</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDecoderState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv">
<span id="_CPPv3NK12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::getDecoderStateC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1aca6ea8e433f9ce18b066b7d1b3d4fa7d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="tensorrt_llm::runtime::decoder"><span class="n"><span class="pre">decoder</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="tensorrt_llm::runtime::decoder::DecoderState"><span class="n"><span class="pre">DecoderState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDecoderState</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getDecoderStreamEv">
<span id="_CPPv3NK12tensorrt_llm7runtime17GptDecoderBatched16getDecoderStreamEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime17GptDecoderBatched16getDecoderStreamEv"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::getDecoderStreamC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a00712a284f039faa4d900c53cceb7326"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDecoderStream</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getDecoderStreamEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched20getUnderlyingDecoderEv">
<span id="_CPPv3NK12tensorrt_llm7runtime17GptDecoderBatched20getUnderlyingDecoderEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime17GptDecoderBatched20getUnderlyingDecoderEv"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::getUnderlyingDecoderC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a682e85aadfb2e29642c5808156752cb0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE" title="tensorrt_llm::runtime::IGptDecoder"><span class="n"><span class="pre">IGptDecoder</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getUnderlyingDecoder</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched20getUnderlyingDecoderEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getBufferManagerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime17GptDecoderBatched16getBufferManagerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime17GptDecoderBatched16getBufferManagerEv"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::getBufferManagerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a8f5e2015905304956ddf68ad64d0b383"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getBufferManager</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getBufferManagerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1af8eb97892b63c439e41ccb09adeb2e37"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GptDecoderPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE" title="tensorrt_llm::runtime::IGptDecoder"><span class="n"><span class="pre">IGptDecoder</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched28setExplicitDraftTokensInputsERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched28setExplicitDraftTokensInputsERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched28setExplicitDraftTokensInputsERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::setExplicitDraftTokensInputs__decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1ab20d2145fd96701343c46c4c7d5177a6"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setExplicitDraftTokensInputs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched28setExplicitDraftTokensInputsERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Sets inputs for explicit draft tokens. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14setEagleInputsERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14setEagleInputsERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14setEagleInputsERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::setEagleInputs__decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a7e170cbad9f4ca2cd84eb3d828e81142"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setEagleInputs</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14setEagleInputsERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Sets inputs for eagle decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15forwardDispatchERN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched15forwardDispatchERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched15forwardDispatchERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::forwardDispatch__decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1accb62cadbd48438f893a0cb99e7a9124"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardDispatch</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15forwardDispatchERN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Calls decoders for tokens per engine step. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14prepareForwardE10SizeType32RN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14prepareForwardE10SizeType32RN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14prepareForwardE10SizeType32RN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::prepareForward__SizeType32.decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a965f5f5bc9be5cda3e08674642360d19"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">prepareForward</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">step</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14prepareForwardE10SizeType32RN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Prepare Input and Output for decoder step. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mRuntimeStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14mRuntimeStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14mRuntimeStreamE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::mRuntimeStream__CudaStreamPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a70dfe7d65b7106a7f67350fb46222c32"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mRuntimeStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mRuntimeStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mDecoderStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14mDecoderStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14mDecoderStreamE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::mDecoderStream__CudaStreamPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1ab9b46b895f196e01eedbe6d9b3206cd4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::CudaStreamPtr"><span class="n"><span class="pre">CudaStreamPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecoderStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mDecoderStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mBufferManagerE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched14mBufferManagerE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched14mBufferManagerE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::mBufferManager__BufferManager"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a6d0d3bf1e75a46a0e124ae68caecb588"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mBufferManager</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mBufferManagerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched8mDecoderE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched8mDecoderE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched8mDecoderE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::mDecoder__GptDecoderPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1acc180102b6c64b88146e253d4070e495"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE" title="tensorrt_llm::runtime::GptDecoderBatched::GptDecoderPtr"><span class="n"><span class="pre">GptDecoderPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecoder</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched8mDecoderE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13mDecoderStateE">
<span id="_CPPv3N12tensorrt_llm7runtime17GptDecoderBatched13mDecoderStateE"></span><span id="_CPPv2N12tensorrt_llm7runtime17GptDecoderBatched13mDecoderStateE"></span><span id="tensorrt_llm::runtime::GptDecoderBatched::mDecoderState__std::shared_ptr:decoder::DecoderState:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GptDecoderBatched_1a693f55c08ae2ab8c333a0ba115df9f48"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="tensorrt_llm::runtime::decoder"><span class="n"><span class="pre">decoder</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="tensorrt_llm::runtime::decoder::DecoderState"><span class="n"><span class="pre">DecoderState</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDecoderState</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13mDecoderStateE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="lookaheadbuffers-h">
<h2>lookaheadBuffers.h<a class="headerlink" href="#lookaheadbuffers-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffersE"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecodingBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1a5344d749f98d1b58a5d3161abf9dcf68"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers24LookaheadDecodingBuffersE10SizeType3210SizeType32RK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers24LookaheadDecodingBuffersE10SizeType3210SizeType32RK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers24LookaheadDecodingBuffersE10SizeType3210SizeType32RK13BufferManager"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers::LookaheadDecodingBuffers__SizeType32.SizeType32.BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1a881786378729c904315c8e50af85f592"></span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecodingBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokensPerStep</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferManager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers24LookaheadDecodingBuffersE10SizeType3210SizeType32RK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers17generationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers17generationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers17generationLengthsE"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers::generationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1a386d5c3e31dd07022de404c4a54aa84a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers17generationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers15positionOffsetsE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers15positionOffsetsE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers15positionOffsetsE"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers::positionOffsets__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1a2e45095b383e0305d81ff601a1cb7587"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionOffsets</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers15positionOffsetsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11packedMasksE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers11packedMasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers11packedMasksE"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers::packedMasks__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1a9ed763d83449eae9909f79dbea9b2cff"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedMasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11packedMasksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11positionIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime24LookaheadDecodingBuffers11positionIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime24LookaheadDecodingBuffers11positionIdsE"></span><span id="tensorrt_llm::runtime::LookaheadDecodingBuffers::positionIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadDecodingBuffers_1aebc8b3c736dd87e008ead3c1f0e81925"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11positionIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffersE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadRuntimeBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a31d09817b403c90e6eb8c2f497e2e888"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a55f6ef4d805bd7fdf28f21cca99f8420"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="tensorrt_llm::runtime::StringPtrMap"><span class="n"><span class="pre">StringPtrMap</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23LookaheadRuntimeBuffersE10SizeType3210SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfigRKN8executor14DecodingConfigERK11TllmRuntime">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23LookaheadRuntimeBuffersE10SizeType3210SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfigRKN8executor14DecodingConfigERK11TllmRuntime"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23LookaheadRuntimeBuffersE10SizeType3210SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfigRKN8executor14DecodingConfigERK11TllmRuntime"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::LookaheadRuntimeBuffers__SizeType32.SizeType32.BufferManagerCR.ModelConfigCR.WorldConfigCR.executor::DecodingConfigCR.TllmRuntimeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1af493b79b5b15ae5928b33dbaa299062d"></span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadRuntimeBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decodingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">TllmRuntime</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">runtime</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23LookaheadRuntimeBuffersE10SizeType3210SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfigRKN8executor14DecodingConfigERK11TllmRuntime" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers13setFromInputsE10SizeType3210SizeType32RK7ITensorRK7ITensorRK24LookaheadDecodingBuffersRK11TllmRuntimeRK11ModelConfigRK11WorldConfig">
<span id="_CPPv3NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers13setFromInputsE10SizeType3210SizeType32RK7ITensorRK7ITensorRK24LookaheadDecodingBuffersRK11TllmRuntimeRK11ModelConfigRK11WorldConfig"></span><span id="_CPPv2NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers13setFromInputsE10SizeType3210SizeType32RK7ITensorRK7ITensorRK24LookaheadDecodingBuffersRK11TllmRuntimeRK11ModelConfigRK11WorldConfig"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::setFromInputs__SizeType32.SizeType32.ITensorCR.ITensorCR.LookaheadDecodingBuffersCR.TllmRuntimeCR.ModelConfigCR.WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ae47ae1982ff0b018e1c59213e1f352cf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFromInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">requestTypes</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers"><span class="n"><span class="pre">LookaheadDecodingBuffers</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decoderLookaheadBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">TllmRuntime</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">runtime</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers13setFromInputsE10SizeType3210SizeType32RK7ITensorRK7ITensorRK24LookaheadDecodingBuffersRK11TllmRuntimeRK11ModelConfigRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers7reshapeE10SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers7reshapeE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers7reshapeE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::reshape__SizeType32.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a07c7f33604a7029e91612644ad8bece5"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokensPerStep</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers7reshapeE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers18insertInputTensorsER9TensorMapR9TensorMapRK11WorldConfig">
<span id="_CPPv3NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers18insertInputTensorsER9TensorMapR9TensorMapRK11WorldConfig"></span><span id="_CPPv2NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers18insertInputTensorsER9TensorMapR9TensorMapRK11WorldConfig"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::insertInputTensors__TensorMapR.TensorMapR.WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a3264e1438be2238bd4d1edbe49883eab"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">insertInputTensors</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">outputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers18insertInputTensorsER9TensorMapR9TensorMapRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23enableLookaheadDecodingE10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23enableLookaheadDecodingE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23enableLookaheadDecodingE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::enableLookaheadDecoding__SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a5b66018ff99cf49b7ac402ae11ce16ce"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">enableLookaheadDecoding</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tokensPerStep</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23enableLookaheadDecodingE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers24disableLookaheadDecodingEv">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers24disableLookaheadDecodingEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers24disableLookaheadDecodingEv"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::disableLookaheadDecoding"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a0d1b68c0a775e0629bbf3fc4d087fc96"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookaheadDecoding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers24disableLookaheadDecodingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers12cumSumLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers12cumSumLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers12cumSumLengthE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::cumSumLength__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ac1dfeec55965185cea1df7528919f64e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumSumLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers12cumSumLengthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17packedMasksDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17packedMasksDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17packedMasksDeviceE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::packedMasksDevice__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1acea94f80db0e0b2a6b39b440f2ed60ec"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedMasksDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17packedMasksDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23generationLengthsDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23generationLengthsDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23generationLengthsDeviceE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::generationLengthsDevice__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a09a19ccefc0db23c5c628004ac72cc1d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengthsDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23generationLengthsDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21positionOffsetsDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21positionOffsetsDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21positionOffsetsDeviceE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionOffsetsDevice__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ab984a78ad4b9b198260bcdd0141b0266"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionOffsetsDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21positionOffsetsDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17positionIdsDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17positionIdsDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17positionIdsDeviceE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionIdsDevice__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a3d0574b682285378c72fbcc7729f1bc7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIdsDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17positionIdsDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers14packedMaskHostE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers14packedMaskHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers14packedMaskHostE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::packedMaskHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ab96b8e1b2a19e4899e58beb4f39d2764"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedMaskHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers14packedMaskHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21generationLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21generationLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21generationLengthsHostE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::generationLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a01cb543b572cc39144170b48cac39266"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21generationLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionOffsetsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionOffsetsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionOffsetsHostE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionOffsetsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a768914b4e84a7f1aed192a9c7ecf99d0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionOffsetsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionOffsetsHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15positionIdsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15positionIdsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15positionIdsHostE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionIdsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a30f45210b0b77d4b824249226749c8cb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIdsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15positionIdsHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18packedMaskHostCopyE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18packedMaskHostCopyE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18packedMaskHostCopyE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::packedMaskHostCopy__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ae004a542179dea5feb713c4f2c2430c3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedMaskHostCopy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18packedMaskHostCopyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers25generationLengthsHostCopyE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers25generationLengthsHostCopyE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers25generationLengthsHostCopyE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::generationLengthsHostCopy__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a55fadcf67070bc31c9691f3655b0da3f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengthsHostCopy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers25generationLengthsHostCopyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23positionOffsetsHostCopyE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23positionOffsetsHostCopyE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23positionOffsetsHostCopyE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionOffsetsHostCopy__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a72bc7457c1183554d7796ffa8e4a1206"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionOffsetsHostCopy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23positionOffsetsHostCopyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionIdsHostCopyE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionIdsHostCopyE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionIdsHostCopyE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::positionIdsHostCopy__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1ae104cf03b3a5625f3e61b99727a768b3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIdsHostCopy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionIdsHostCopyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15useSpecDecodingE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15useSpecDecodingE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15useSpecDecodingE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::useSpecDecoding__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a2500579fce4262a16bcbc68b77b615f9"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useSpecDecoding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15useSpecDecodingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18batchSlotsHostCopyE">
<span id="_CPPv3N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18batchSlotsHostCopyE"></span><span id="_CPPv2N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18batchSlotsHostCopyE"></span><span id="tensorrt_llm::runtime::LookaheadRuntimeBuffers::batchSlotsHostCopy__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LookaheadRuntimeBuffers_1a153cb9a02883c543e4779d20cfcdb72b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE" title="tensorrt_llm::runtime::LookaheadRuntimeBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchSlotsHostCopy</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18batchSlotsHostCopyE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="loramodule-h">
<h2>loraModule.h<a class="headerlink" href="#loramodule-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK10LoraModule">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERK10LoraModule"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERK10LoraModule"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.LoraModuleCR"></span><span class="target" id="loraModule_8h_1a019fd70ba84e9b865bc6b7b58db3fd6f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">module</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK10LoraModule" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModuleE"></span><span id="tensorrt_llm::runtime::LoraModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraModule</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ModuleType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType8kINVALIDE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType8kINVALIDE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType8kINVALIDE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a90e668d1015c46c274bdcf183d2bfd30"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kINVALID</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType8kINVALIDE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kATTN_QKVE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType9kATTN_QKVE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType9kATTN_QKVE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a3abaa6d49d41464821a96fe45ae499f3"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTN_QKV</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kATTN_QKVE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_QE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_QE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_QE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a885b263f8c935fc2e9a227e7fb7bfc85"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTN_Q</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_QE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_KE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_KE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_KE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a1d15211064de5ce58d9aa8b3f54130f2"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTN_K</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_KE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_VE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_VE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_VE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8aed04252974b4c95c0691bbbd7d554557"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTN_V</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_VE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kATTN_DENSEE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType11kATTN_DENSEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType11kATTN_DENSEE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a9678d862c1ecefc4790acc3076a9ae39"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kATTN_DENSE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kATTN_DENSEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_H_TO_4HE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_H_TO_4HE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_H_TO_4HE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a5897d380b701b8c819f6452783ea76a6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLP_H_TO_4H</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_H_TO_4HE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_4H_TO_HE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_4H_TO_HE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_4H_TO_HE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a1291a3a4a4726a507e30b07cecc754b8"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLP_4H_TO_H</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_4H_TO_HE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMLP_GATEE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType9kMLP_GATEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType9kMLP_GATEE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a5414fc337719904c181fb53f649a6a02"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLP_GATE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMLP_GATEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType15kCROSS_ATTN_QKVE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType15kCROSS_ATTN_QKVE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType15kCROSS_ATTN_QKVE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a7b69854e31937cdceb210086feefea65"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCROSS_ATTN_QKV</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType15kCROSS_ATTN_QKVE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_QE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_QE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_QE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a82f7bdb5649b5d0f8a90fa1ebb5c1839"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCROSS_ATTN_Q</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_QE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_KE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_KE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_KE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a6a2b7e8676549d9de16f408efc6a2614"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCROSS_ATTN_K</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_KE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_VE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_VE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_VE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a0b7fee7135c586e02ea30c3b61e59c7c"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCROSS_ATTN_V</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_VE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType17kCROSS_ATTN_DENSEE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType17kCROSS_ATTN_DENSEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType17kCROSS_ATTN_DENSEE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a4831dc4fe8b758988c399d8b3368fcd4"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCROSS_ATTN_DENSE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType17kCROSS_ATTN_DENSEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_H_TO_4HE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_H_TO_4HE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_H_TO_4HE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a3f915fd485fd5c23d418effadcdef87c"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMOE_H_TO_4H</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_H_TO_4HE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_4H_TO_HE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_4H_TO_HE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_4H_TO_HE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a16cb51bf6af53b8657c999409aa1c5e8"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMOE_4H_TO_H</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_4H_TO_HE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMOE_GATEE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType9kMOE_GATEE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType9kMOE_GATEE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a853eac337a8fbded3c1ffffe156cbc8e"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMOE_GATE</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMOE_GATEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMOE_ROUTERE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType11kMOE_ROUTERE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType11kMOE_ROUTERE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a743f330fc64cb3653e638bdd503ad2b3"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMOE_ROUTER</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMOE_ROUTERE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMLP_ROUTERE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType11kMLP_ROUTERE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType11kMLP_ROUTERE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a21e30455fd5abaff942fb59aefd875c4"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLP_ROUTER</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMLP_ROUTERE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_GATE_UPE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_GATE_UPE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_GATE_UPE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a784821fbc7f44239dcf33e75bad249b8a941ac1ef5486393e4fc0129d27d4ebd9"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMLP_GATE_UP</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_GATE_UPE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a894d3a0fd7f335adb87a0f4d8efe672a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10ModuleType10SizeType3210SizeType32bb10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10ModuleType10SizeType3210SizeType32bb10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10ModuleType10SizeType3210SizeType32bb10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::LoraModule__ModuleTypeCR.SizeType32.SizeType32.b.b.SizeType32.SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1acb1372f4c7079235fb8e594691919fee"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE" title="tensorrt_llm::runtime::LoraModule::ModuleType"><span class="n"><span class="pre">ModuleType</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">t</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">inDim</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">outDim</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">inDimFirst</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">outDimFirst</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">inTpSplitDim</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">outTpSplitDim</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10ModuleType10SizeType3210SizeType32bb10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10LoraModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10LoraModuleEv"></span><span id="tensorrt_llm::runtime::LoraModule::LoraModuleCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a010f32ba204263e4ce58494e02ea9198"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule"></span><span id="tensorrt_llm::runtime::LoraModule::LoraModule__LoraModuleCRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ade6c5f359962e48848102880cfa72fa2"></span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraModule</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule" title="tensorrt_llm::runtime::LoraModule::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModuleaSERK10LoraModule">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModuleaSERK10LoraModule"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModuleaSERK10LoraModule"></span><span id="tensorrt_llm::runtime::LoraModule::assign-operator__LoraModuleCRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a0318026e4d60431ed806daa1807dcc23"></span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleaSERK10LoraModule" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule18flattenedInOutSizeE10SizeType32b">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule18flattenedInOutSizeE10SizeType32b"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule18flattenedInOutSizeE10SizeType32b"></span><span id="tensorrt_llm::runtime::LoraModule::flattenedInOutSize__SizeType32.bCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a98faaa62a1b80ff7c13f06b9b47f500c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">flattenedInOutSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isDora</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule18flattenedInOutSizeE10SizeType32b" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule6inSizeE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule6inSizeE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule6inSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::inSize__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a4cbb632fe6af3a63048df1392aaabaa3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule6inSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule7outSizeE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule7outSizeE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule7outSizeE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::outSize__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ad7cf8314d5f8fb50d8dc4c10ea778674"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule7outSizeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule11localInSizeE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule11localInSizeE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule11localInSizeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localInSize__SizeType32.SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a59babff6e5dea96b528bc6be8e42a28f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localInSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11localInSizeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule12localOutSizeE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule12localOutSizeE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule12localOutSizeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localOutSize__SizeType32.SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a3e6970a72ca8287ce0c4baac39cf9497"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localOutSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule12localOutSizeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule15localScalesSizeE10SizeType32b">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule15localScalesSizeE10SizeType32b"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule15localScalesSizeE10SizeType32b"></span><span id="tensorrt_llm::runtime::LoraModule::localScalesSize__SizeType32.bCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a0b6ff71c782cca7698b5e7f36d6192b3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localScalesSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isDora</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule15localScalesSizeE10SizeType32b" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule10localInDimE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule10localInDimE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule10localInDimE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localInDim__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ac86684c323e7e21d1b46964ce68580e4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localInDim</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule10localInDimE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule11localOutDimE10SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule11localOutDimE10SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule11localOutDimE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localOutDim__SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a7f5d2a328ee5e0b45e82053e13016168"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localOutDim</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11localOutDimE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule18localInAdapterSizeE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule18localInAdapterSizeE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule18localInAdapterSizeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localInAdapterSize__SizeType32.SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a71b480241e25d9d3a6b33c444f790bff"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localInAdapterSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule18localInAdapterSizeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule19localOutAdapterSizeE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule19localOutAdapterSizeE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule19localOutAdapterSizeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localOutAdapterSize__SizeType32.SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1aa8d74dff461cca4c35017e85bafd6078"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localOutAdapterSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule19localOutAdapterSizeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule14localInOutSizeE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule14localInOutSizeE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule14localInOutSizeE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::localInOutSize__SizeType32.SizeType32CCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a6884e16ab6f2d35329f34a8194870496"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localInOutSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule14localInOutSizeE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule14localTotalSizeE10SizeType3210SizeType32b">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule14localTotalSizeE10SizeType3210SizeType32b"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule14localTotalSizeE10SizeType3210SizeType32b"></span><span id="tensorrt_llm::runtime::LoraModule::localTotalSize__SizeType32.SizeType32.bCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a357dda229ed76216605d0769c8fb92a4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">localTotalSize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">adapterSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">isDora</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule14localTotalSizeE10SizeType3210SizeType32b" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule5valueEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule5valueEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule5valueEv"></span><span id="tensorrt_llm::runtime::LoraModule::valueCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ac3fa242daa4252b826531467fda176f2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule5valueEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule4nameEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule4nameEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule4nameEv"></span><span id="tensorrt_llm::runtime::LoraModule::nameCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a6d3d51b4414933c97d3e5a64d08e7d0f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string_view</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule4nameEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule5inDimEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule5inDimEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule5inDimEv"></span><span id="tensorrt_llm::runtime::LoraModule::inDimCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a221a2ce81aafff9feae5fd3a7785b3a4"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inDim</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule5inDimEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule6outDimEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule6outDimEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule6outDimEv"></span><span id="tensorrt_llm::runtime::LoraModule::outDimCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a638a5cfeb52c2343106ba7e700a2abd6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outDim</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule6outDimEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule10inDimFirstEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule10inDimFirstEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule10inDimFirstEv"></span><span id="tensorrt_llm::runtime::LoraModule::inDimFirstCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ac380116029394b465e212009b2151e16"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inDimFirst</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule10inDimFirstEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule11outDimFirstEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule11outDimFirstEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule11outDimFirstEv"></span><span id="tensorrt_llm::runtime::LoraModule::outDimFirstCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a30b637769120876b91d81f93fcf4fcc1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outDimFirst</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11outDimFirstEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule12inTpSplitDimEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule12inTpSplitDimEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule12inTpSplitDimEv"></span><span id="tensorrt_llm::runtime::LoraModule::inTpSplitDimCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ad81da0b83ac2896ea83bef6edbe5c301"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inTpSplitDim</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule12inTpSplitDimEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10LoraModule13outTpSplitDimEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10LoraModule13outTpSplitDimEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10LoraModule13outTpSplitDimEv"></span><span id="tensorrt_llm::runtime::LoraModule::outTpSplitDimCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1aee2e5e7995d7445e61e99466a0671c6c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outTpSplitDim</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule13outTpSplitDimEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule17createLoraModulesERKNSt6vectorINSt6stringEEE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule17createLoraModulesERKNSt6vectorINSt6stringEEE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule17createLoraModulesERKNSt6vectorINSt6stringEEE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::createLoraModules__std::vector:ss:CR.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a93b14a632d0f904290d2a7e9d0debf46"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE" title="tensorrt_llm::runtime::LoraModule"><span class="n"><span class="pre">LoraModule</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">createLoraModules</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">loraModuleNames</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">hiddenSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">mlpHiddenSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numAttentionHeads</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numKvAttentionHeads</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">attentionHeadSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tpSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numExperts</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule17createLoraModulesERKNSt6vectorINSt6stringEEE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleTypeERKNSt11string_viewE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule12toModuleTypeERKNSt11string_viewE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule12toModuleTypeERKNSt11string_viewE"></span><span id="tensorrt_llm::runtime::LoraModule::toModuleType__std::string_viewCRCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ab8265ca2adf436d70b3ca2309fe66c96"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE" title="tensorrt_llm::runtime::LoraModule::ModuleType"><span class="n"><span class="pre">ModuleType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toModuleType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string_view</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">name</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleTypeERKNSt11string_viewE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10ModuleType">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule12toModuleNameE10ModuleType"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule12toModuleNameE10ModuleType"></span><span id="tensorrt_llm::runtime::LoraModule::toModuleName__ModuleTypeCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a4b2ba83d368ecf7cc04a9554552939a3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string_view</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toModuleName</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE" title="tensorrt_llm::runtime::LoraModule::ModuleType"><span class="n"><span class="pre">ModuleType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">t</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10ModuleType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule12toModuleNameE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule12toModuleNameE10SizeType32"></span><span id="tensorrt_llm::runtime::LoraModule::toModuleName__SizeType32CE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a3767755d14fbb821ddbbb2d353c54d33"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string_view</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toModuleName</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">id</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule5mTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule5mTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule5mTypeE"></span><span id="tensorrt_llm::runtime::LoraModule::mType__ModuleType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1ae7b52971db55bf920e68ce2d246767e8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE" title="tensorrt_llm::runtime::LoraModule::ModuleType"><span class="n"><span class="pre">ModuleType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule5mTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule6mInDimE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule6mInDimE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule6mInDimE"></span><span id="tensorrt_llm::runtime::LoraModule::mInDim__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a7bb2cb44338e43391db5183d89592547"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInDim</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule6mInDimE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule7mOutDimE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule7mOutDimE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule7mOutDimE"></span><span id="tensorrt_llm::runtime::LoraModule::mOutDim__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a87596bc34d693acff958dddc44d45f49"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOutDim</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule7mOutDimE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule11mInDimFirstE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule11mInDimFirstE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule11mInDimFirstE"></span><span id="tensorrt_llm::runtime::LoraModule::mInDimFirst__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1aee316e1b2b3fb3cc31c1f94c6dcea07f"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInDimFirst</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule11mInDimFirstE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule12mOutDimFirstE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule12mOutDimFirstE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule12mOutDimFirstE"></span><span id="tensorrt_llm::runtime::LoraModule::mOutDimFirst__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1aa04960b7f5fac8ae65016db25dd8f64c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOutDimFirst</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12mOutDimFirstE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule13mInTpSplitDimE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule13mInTpSplitDimE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule13mInTpSplitDimE"></span><span id="tensorrt_llm::runtime::LoraModule::mInTpSplitDim__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a85e27922cbd350fc4b21358d4f43b389"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mInTpSplitDim</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule13mInTpSplitDimE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10LoraModule14mOutTpSplitDimE">
<span id="_CPPv3N12tensorrt_llm7runtime10LoraModule14mOutTpSplitDimE"></span><span id="_CPPv2N12tensorrt_llm7runtime10LoraModule14mOutTpSplitDimE"></span><span id="tensorrt_llm::runtime::LoraModule::mOutTpSplitDim__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1LoraModule_1a82af046608a063edd7aeda7898de377f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOutTpSplitDim</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule14mOutTpSplitDimE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="prompttuningparams-h">
<h2>promptTuningParams.h<a class="headerlink" href="#prompttuningparams-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TTensor</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GenericPromptTuningParams</span></span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1a7f0226e537f0c4a164c298adf9731e67"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TTensor"><span class="n"><span class="pre">TTensor</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1a91aed2fabb50029ec34af4a22a952b77"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams25GenericPromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams25GenericPromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams25GenericPromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"></span><span id="tensorrt_llm::runtime::GenericPromptTuningParams::GenericPromptTuningParams__TensorPtr.TensorPtr.TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1a08b54a1a315a75cada66405e31709668"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">GenericPromptTuningParams</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">embeddingTable</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tasks</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams25GenericPromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams14embeddingTableE">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams14embeddingTableE"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams14embeddingTableE"></span><span id="tensorrt_llm::runtime::GenericPromptTuningParams::embeddingTable__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1a718fcc98779017166cbe0a748c081414"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">embeddingTable</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams14embeddingTableE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams5tasksE">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams5tasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams5tasksE"></span><span id="tensorrt_llm::runtime::GenericPromptTuningParams::tasks__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1aea7aafec3a272782a1a467eb159af6e1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams5tasksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9vocabSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams9vocabSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams9vocabSizeE"></span><span id="tensorrt_llm::runtime::GenericPromptTuningParams::vocabSize__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1acb732cd787508ea7013cf1e9002590ea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::GenericPromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">vocabSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9vocabSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams19promptTuningEnabledE">
<span id="_CPPv3N12tensorrt_llm7runtime25GenericPromptTuningParams19promptTuningEnabledE"></span><span id="_CPPv2N12tensorrt_llm7runtime25GenericPromptTuningParams19promptTuningEnabledE"></span><span id="tensorrt_llm::runtime::GenericPromptTuningParams::promptTuningEnabled__std::vector:b:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1GenericPromptTuningParams_1a4f994aa4122125396dee70846aec1318"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">promptTuningEnabled</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams19promptTuningEnabledE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18PromptTuningParamsE">
<span id="_CPPv3N12tensorrt_llm7runtime18PromptTuningParamsE"></span><span id="_CPPv2N12tensorrt_llm7runtime18PromptTuningParamsE"></span><span id="tensorrt_llm::runtime::PromptTuningParams"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1PromptTuningParams"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PromptTuningParams</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE" title="tensorrt_llm::runtime::GenericPromptTuningParams"><span class="n"><span class="pre">GenericPromptTuningParams</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParamsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1PromptTuningParams_1acf57960de488b9fcc3021a34690f003a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1PromptTuningParams_1a99d9372e247512429412043f3103e08a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE" title="tensorrt_llm::runtime::GenericPromptTuningParams"><span class="n"><span class="pre">GenericPromptTuningParams</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E" title="tensorrt_llm::runtime::GenericPromptTuningParams::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18PromptTuningParams18PromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime18PromptTuningParams18PromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime18PromptTuningParams18PromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"></span><span id="tensorrt_llm::runtime::PromptTuningParams::PromptTuningParams__TensorPtr.TensorPtr.TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1PromptTuningParams_1a6439bfa322d7dc1f589cd3877640a952"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PromptTuningParams</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::PromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">embeddingTable</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::PromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tasks</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::PromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSize</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">nullptr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams18PromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18PromptTuningParams15fillTasksTensorE9TensorPtr10SizeType3210SizeType32RKNSt6vectorI10SizeType32EERKNSt6vectorI10SizeType32EERK13BufferManagerb">
<span id="_CPPv3N12tensorrt_llm7runtime18PromptTuningParams15fillTasksTensorE9TensorPtr10SizeType3210SizeType32RKNSt6vectorI10SizeType32EERKNSt6vectorI10SizeType32EERK13BufferManagerb"></span><span id="_CPPv2N12tensorrt_llm7runtime18PromptTuningParams15fillTasksTensorE9TensorPtr10SizeType3210SizeType32RKNSt6vectorI10SizeType32EERKNSt6vectorI10SizeType32EERK13BufferManagerb"></span><span id="tensorrt_llm::runtime::PromptTuningParams::fillTasksTensor__TensorPtr.SizeType32.SizeType32.std::vector:SizeType32:CR.std::vector:SizeType32:CR.BufferManagerCR.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1PromptTuningParams_1a6caea9d5d0afa0de13bb9a30a7cadcaf"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">fillTasksTensor</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE" title="tensorrt_llm::runtime::PromptTuningParams::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tasksHost</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E" title="tensorrt_llm::runtime::PromptTuningParams::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E" title="tensorrt_llm::runtime::PromptTuningParams::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numContextRequests</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E" title="tensorrt_llm::runtime::PromptTuningParams::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">reqBeamWidths</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E" title="tensorrt_llm::runtime::PromptTuningParams::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">reqPromptLengths</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">packedInput</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams15fillTasksTensorE9TensorPtr10SizeType3210SizeType32RKNSt6vectorI10SizeType32EERKNSt6vectorI10SizeType32EERK13BufferManagerb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="speculativedecodingmode-h">
<h2>speculativeDecodingMode.h<a class="headerlink" href="#speculativedecodingmode-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingModeE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingModeE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a142fe60d488053b88f9961e51993cd4c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UnderlyingType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode6isNoneEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode6isNoneEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode6isNoneEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isNoneCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1acd3c978d723e3a4888d10f06f71adab6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isNone</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode6isNoneEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isDraftTokensExternalEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isDraftTokensExternalEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isDraftTokensExternalEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isDraftTokensExternalCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1afee189c9b551928bb2645c14a8063871"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isDraftTokensExternal</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isDraftTokensExternalEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode8isMedusaEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode8isMedusaEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode8isMedusaEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isMedusaCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1ac386b84b5b4d90fd2bcc311514428c4d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isMedusa</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode8isMedusaEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19isLookaheadDecodingEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode19isLookaheadDecodingEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode19isLookaheadDecodingEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isLookaheadDecodingCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a14e479f01ad1d809786603b6f1265b0f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isLookaheadDecoding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19isLookaheadDecodingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isExplicitDraftTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isExplicitDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isExplicitDraftTokensEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isExplicitDraftTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1aca96f1a5e256bc1e851819c44825ae02"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isExplicitDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isExplicitDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode7isEagleEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode7isEagleEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode7isEagleEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::isEagleCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a96e3d17ad9aa9a93df5e2e8cf029710a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isEagle</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode7isEagleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18updatesPositionIdsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode18updatesPositionIdsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode18updatesPositionIdsEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::updatesPositionIdsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a3ebdee2a9c4aebd54efa347ac4b48d33"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">updatesPositionIds</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18updatesPositionIdsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21requiresAttentionMaskEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode21requiresAttentionMaskEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode21requiresAttentionMaskEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::requiresAttentionMaskCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1adc3b3407372f26c7caf42f09ad5457a8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">requiresAttentionMask</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21requiresAttentionMaskEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19predictsDraftTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode19predictsDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode19predictsDraftTokensEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::predictsDraftTokensCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a546d33fe11fea48eea7242d4d4279060"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">predictsDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19predictsDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18needsKVCacheRewindEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode18needsKVCacheRewindEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode18needsKVCacheRewindEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::needsKVCacheRewindCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1adc46b81262fd8e85146857dee395a438"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">needsKVCacheRewind</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18needsKVCacheRewindEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19variableDraftLengthEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode19variableDraftLengthEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode19variableDraftLengthEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::variableDraftLengthCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a1fbbcb52a29d01aa4326fb6587502539"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">variableDraftLength</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19variableDraftLengthEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode14hasDraftLogitsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode14hasDraftLogitsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode14hasDraftLogitsEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::hasDraftLogitsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a013f7fdcadf107956d33adcde8ad38f2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">hasDraftLogits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode14hasDraftLogitsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode20needsDecoderPrologueEv">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode20needsDecoderPrologueEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode20needsDecoderPrologueEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::needsDecoderPrologueCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a9153eb4a918de5a7e093e426888d3986"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">needsDecoderPrologue</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode20needsDecoderPrologueEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingModeeqERK23SpeculativeDecodingMode">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingModeeqERK23SpeculativeDecodingMode"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingModeeqERK23SpeculativeDecodingMode"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::eq-operator__SpeculativeDecodingModeCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a545625bd71856b9ed609b9424ad09fef"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingModeeqERK23SpeculativeDecodingMode" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode23SpeculativeDecodingModeE14UnderlyingType">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode23SpeculativeDecodingModeE14UnderlyingType"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode23SpeculativeDecodingModeE14UnderlyingType"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::SpeculativeDecodingMode__UnderlyingTypeCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a27fe9396ebb4470673dafa60eecf6db5"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingMode</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">state</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode23SpeculativeDecodingModeE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::NoneCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a0e993a88f1f7fd5b110459fb9aef8142"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">None</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19DraftTokensExternalEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode19DraftTokensExternalEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode19DraftTokensExternalEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::DraftTokensExternalCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a34e2a084be1ba4dc7f1fddba221bb9df"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DraftTokensExternal</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19DraftTokensExternalEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6MedusaEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode6MedusaEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode6MedusaEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::MedusaCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a37730089e251e3ccd1e6e50d3ac2dead"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Medusa</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6MedusaEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode17LookaheadDecodingEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode17LookaheadDecodingEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode17LookaheadDecodingEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::LookaheadDecodingCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a87b917526f7f2b0d821f7e034610649c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LookaheadDecoding</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode17LookaheadDecodingEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19ExplicitDraftTokensEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode19ExplicitDraftTokensEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode19ExplicitDraftTokensEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::ExplicitDraftTokensCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1ae7738d4022d1a16bbde026f7ae69acbf"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExplicitDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19ExplicitDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5EagleEv">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode5EagleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode5EagleEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::EagleCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a18204f24dba152c9fc208659f7e97a3b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Eagle</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5EagleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9anyBitSetE14UnderlyingType">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode9anyBitSetE14UnderlyingType"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode9anyBitSetE14UnderlyingType"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::anyBitSet__UnderlyingTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a0a63f2f97f693e4c860330753711cdd6"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">anyBitSet</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bits</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9anyBitSetE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9allBitSetE14UnderlyingType">
<span id="_CPPv3NK12tensorrt_llm7runtime23SpeculativeDecodingMode9allBitSetE14UnderlyingType"></span><span id="_CPPv2NK12tensorrt_llm7runtime23SpeculativeDecodingMode9allBitSetE14UnderlyingType"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::allBitSet__UnderlyingTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1abc67823d81ebe2d45fbdbd7908e11153"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allBitSet</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">bits</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9allBitSetE14UnderlyingType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6mStateE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode6mStateE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode6mStateE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::mState__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a0e8510f7bd29689984bc0cea9bff334f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mState</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::kNone"><span class="n"><span class="pre">kNone</span></span></a><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6mStateE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-attributes">Private Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kNone__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1af78fd605d8b1f0fca8f5dbf4beb1618b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kNone</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">0U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kDraftTokensExternalE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode20kDraftTokensExternalE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode20kDraftTokensExternalE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kDraftTokensExternal__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a096339698fc534bad97d16e3b044c461"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kDraftTokensExternal</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">1U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kDraftTokensExternalE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode7kMedusaE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode7kMedusaE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode7kMedusaE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kMedusa__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1af315abd0c172f828f74b9e17abbd903f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kMedusa</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">2U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode7kMedusaE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode18kLookaheadDecodingE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode18kLookaheadDecodingE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode18kLookaheadDecodingE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kLookaheadDecoding__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1ae9223f63d456516b693af04eed4b1178"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kLookaheadDecoding</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">3U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode18kLookaheadDecodingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kExplicitDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode20kExplicitDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode20kExplicitDraftTokensE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kExplicitDraftTokens__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1a7c8f3d5f099d84183f49969066c998da"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kExplicitDraftTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">4U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kExplicitDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6kEagleE">
<span id="_CPPv3N12tensorrt_llm7runtime23SpeculativeDecodingMode6kEagleE"></span><span id="_CPPv2N12tensorrt_llm7runtime23SpeculativeDecodingMode6kEagleE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingMode::kEagle__UnderlyingType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingMode_1acf0f91631415d19f3b8cff019a1faf41"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE" title="tensorrt_llm::runtime::SpeculativeDecodingMode::UnderlyingType"><span class="n"><span class="pre">UnderlyingType</span></span></a><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kEagle</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1U</span></span><span class="w"> </span><span class="o"><span class="pre">&lt;&lt;</span></span><span class="w"> </span><span class="m"><span class="pre">5U</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6kEagleE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="common-h">
<h2>common.h<a class="headerlink" href="#common-h" title="Link to this heading">#</a></h2>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-defines">Defines</p>
<dl class="cpp macro">
<dt class="sig sig-object cpp" id="c.FMT_DIM">
<span class="target" id="common_8h_1a510c0e5d6315b189e4726c3dd6a76271"></span><span class="sig-name descname"><span class="n"><span class="pre">FMT_DIM</span></span></span><a class="headerlink" href="#c.FMT_DIM" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime10SizeType32E"></span><span class="target" id="common_8h_1a3bdb407122ee5c03962b4aea2f6c61e8"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10SizeType64E">
<span id="_CPPv3N12tensorrt_llm7runtime10SizeType64E"></span><span id="_CPPv2N12tensorrt_llm7runtime10SizeType64E"></span><span class="target" id="common_8h_1aca6041db22beb41ce3c4640c45058773"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10SizeType64E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11TokenIdTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TokenIdTypeE"></span><span class="target" id="common_8h_1a1de916f1c3f3c8d1f9c66320afc6df17"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TokenIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14LoraTaskIdTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14LoraTaskIdTypeE"></span><span class="target" id="common_8h_1aa4d6a559b4a19f8fbab65e8e7a0e69fe"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LoraTaskIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16TokenExtraIdTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime16TokenExtraIdTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime16TokenExtraIdTypeE"></span><span class="target" id="common_8h_1a6fbec83529ee7fceb176b465d97f5d6e"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TokenExtraIdType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16TokenExtraIdTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16VecTokenExtraIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime16VecTokenExtraIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime16VecTokenExtraIdsE"></span><span class="target" id="common_8h_1a4df1b36fecce49a24d250a14ae2b7d85"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">VecTokenExtraIds</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime16TokenExtraIdTypeE" title="tensorrt_llm::runtime::TokenExtraIdType"><span class="n"><span class="pre">TokenExtraIdType</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16VecTokenExtraIdsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime15VecUniqueTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime15VecUniqueTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime15VecUniqueTokensE"></span><span class="target" id="common_8h_1a119cacfef2e257e99f248ee75116134c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">VecUniqueTokens</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="tensorrt_llm::runtime::UniqueToken"><span class="n"><span class="pre">UniqueToken</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime15VecUniqueTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime12StringPtrMapE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime12StringPtrMapE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="common_8h_1a43946c471b82feb36a6350de9cde277d"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StringPtrMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unordered_map</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="tensorrt_llm::runtime::StringPtrMap::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-enums">Enums</p>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11RequestTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11RequestTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11RequestTypeE"></span><span class="target" id="common_8h_1a919e1e7b6860dffaef8f53660c992ca0"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11RequestTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11RequestType8kCONTEXTE">
<span id="_CPPv3N12tensorrt_llm7runtime11RequestType8kCONTEXTE"></span><span id="_CPPv2N12tensorrt_llm7runtime11RequestType8kCONTEXTE"></span><span class="target" id="common_8h_1a919e1e7b6860dffaef8f53660c992ca0ab313e6f758ff978c83b115d402efb5f0"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCONTEXT</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11RequestType8kCONTEXTE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11RequestType11kGENERATIONE">
<span id="_CPPv3N12tensorrt_llm7runtime11RequestType11kGENERATIONE"></span><span id="_CPPv2N12tensorrt_llm7runtime11RequestType11kGENERATIONE"></span><span class="target" id="common_8h_1a919e1e7b6860dffaef8f53660c992ca0ad3fecc7bf972e65c8bc64551251be711"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGENERATION</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11RequestType11kGENERATIONE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
</div>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11UniqueTokenE">
<span id="_CPPv3N12tensorrt_llm7runtime11UniqueTokenE"></span><span id="_CPPv2N12tensorrt_llm7runtime11UniqueTokenE"></span><span id="tensorrt_llm::runtime::UniqueToken"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1UniqueToken"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UniqueToken</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime11UniqueTokeneqERK11UniqueToken">
<span id="_CPPv3NK12tensorrt_llm7runtime11UniqueTokeneqERK11UniqueToken"></span><span id="_CPPv2NK12tensorrt_llm7runtime11UniqueTokeneqERK11UniqueToken"></span><span id="tensorrt_llm::runtime::UniqueToken::eq-operator__UniqueTokenCRC"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1UniqueToken_1ac13480743f9ec9bb14da311d96ed9536"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">==</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE" title="tensorrt_llm::runtime::UniqueToken"><span class="n"><span class="pre">UniqueToken</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime11UniqueTokeneqERK11UniqueToken" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11UniqueToken7tokenIdE">
<span id="_CPPv3N12tensorrt_llm7runtime11UniqueToken7tokenIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime11UniqueToken7tokenIdE"></span><span id="tensorrt_llm::runtime::UniqueToken::tokenId__TokenIdType"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1UniqueToken_1a126d61de5902884d3a08a85a502b4afc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE" title="tensorrt_llm::runtime::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokenId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11UniqueToken7tokenIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11UniqueToken12tokenExtraIdE">
<span id="_CPPv3N12tensorrt_llm7runtime11UniqueToken12tokenExtraIdE"></span><span id="_CPPv2N12tensorrt_llm7runtime11UniqueToken12tokenExtraIdE"></span><span id="tensorrt_llm::runtime::UniqueToken::tokenExtraId__TokenExtraIdType"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1UniqueToken_1af5dc28b61aa34a4ae8a01f85695bfdd3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime16TokenExtraIdTypeE" title="tensorrt_llm::runtime::TokenExtraIdType"><span class="n"><span class="pre">TokenExtraIdType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tokenExtraId</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11UniqueToken12tokenExtraIdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="medusamodule-h">
<h2>medusaModule.h<a class="headerlink" href="#medusamodule-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModuleE"></span><span id="tensorrt_llm::runtime::MedusaModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaModule</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModuleE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModule9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModule9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModule9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1a4ad80835c2e8efd62e459d3cacbc252a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1a811960804e1abe84290dc2d9ef1048df"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaChoices</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleE10SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModule12MedusaModuleE10SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModule12MedusaModuleE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::MedusaModule::MedusaModule__SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1a90e71c121e29000fcedc0a4f69b1191e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAcceptedTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftTokens</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModule12MedusaModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModule12MedusaModuleEv"></span><span id="tensorrt_llm::runtime::MedusaModule::MedusaModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1acd250003687dc783ce61c6d5e03f2da9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MedusaModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime12MedusaModule16getMedusaChoicesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime12MedusaModule16getMedusaChoicesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime12MedusaModule16getMedusaChoicesEv"></span><span id="tensorrt_llm::runtime::MedusaModule::getMedusaChoicesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1a59c59e09ca553ababc8b4088505160f7"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE" title="tensorrt_llm::runtime::MedusaModule::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getMedusaChoices</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime12MedusaModule16getMedusaChoicesEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12MedusaModule21mDefaultMedusaChoicesE">
<span id="_CPPv3N12tensorrt_llm7runtime12MedusaModule21mDefaultMedusaChoicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime12MedusaModule21mDefaultMedusaChoicesE"></span><span id="tensorrt_llm::runtime::MedusaModule::mDefaultMedusaChoices__MedusaChoices"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1MedusaModule_1aea2c515147409dd184931a4a1fa9ace1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE" title="tensorrt_llm::runtime::MedusaModule::MedusaChoices"><span class="n"><span class="pre">MedusaChoices</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDefaultMedusaChoices</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">4</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">3</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">8</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">5</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">2</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">9</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">6</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">7</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule21mDefaultMedusaChoicesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="decodingoutput-h">
<h2>decodingOutput.h<a class="headerlink" href="#decodingoutput-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1batch__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batch_manager</span></span></span><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutputE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutputE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutputE"></span><span id="tensorrt_llm::runtime::DecodingOutput"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingOutput</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1aeb215aae60278c44bcdbd17ae5f7c8e1"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14DecodingOutputE9TensorPtr9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14DecodingOutputE9TensorPtr9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14DecodingOutputE9TensorPtr9TensorPtr"></span><span id="tensorrt_llm::runtime::DecodingOutput::DecodingOutput__TensorPtr.TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1af68b04c734d41e77cea6854abd7bdc2c"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DecodingOutput</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">ids</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">gatheredIds</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14DecodingOutputE9TensorPtr9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput3idsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput3idsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput3idsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::ids__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a9937790aaf8c5cbb5230236afc7656be"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ids</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput3idsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Mandatory parameters Previously generated token ids for all steps before <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1DecodingInput_1a41c49960bea968b4c4e6b7d073c57769"><span class="std std-ref">DecodingInput.step</span></a>, [BS, BM, MSL] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput11gatheredIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput11gatheredIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput11gatheredIdsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::gatheredIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a7694f98fca6aed5f97e731217d97ff50"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">gatheredIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11gatheredIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>The tokens computed during the gatherTree step, [BS, BM, MSL] Necessary for “Streaming + Beam Search” mode since beam search kernels store ungathered tokens in <code class="docutils literal notranslate"><span class="pre">ids</span></code>. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14newTokensStepsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14newTokensStepsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14newTokensStepsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::newTokensSteps__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a6e42e3b0fd2b24ba8a04f78f837b207a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">newTokensSteps</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14newTokensStepsE" title="Link to this definition">#</a><br /></dt>
<dd><p>New tokens at each generated token of maxTokensPerStep, [maxTokensPerStep, BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput9newTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput9newTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput9newTokensE"></span><span id="tensorrt_llm::runtime::DecodingOutput::newTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a1c92d23772ed90b78f9ac8c86f46d9fd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">newTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9newTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>A view of newTokensSteps for the current token, [BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput12newTokensVecE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput12newTokensVecE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput12newTokensVecE"></span><span id="tensorrt_llm::runtime::DecodingOutput::newTokensVec__std::vector:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a0df2485f70ab5959e706840f7635e4c3"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">newTokensVec</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput12newTokensVecE" title="Link to this definition">#</a><br /></dt>
<dd><p>A Vector of views on newTokensSteps for each token [BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput13finishReasonsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput13finishReasonsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput13finishReasonsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::finishReasons__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a1b520adda489db15ba1b8fde1baa195f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finishReasons</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput13finishReasonsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Optional parameters FinishedState by decoding if any of the stop conditions are met or if DecodingInput.finished is true, [BS, BM] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput11finishedSumE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput11finishedSumE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput11finishedSumE"></span><span id="tensorrt_llm::runtime::DecodingOutput::finishedSum__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a2bb2c2cc930598e59363a8d61e2f0fc3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finishedSum</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11finishedSumE" title="Link to this definition">#</a><br /></dt>
<dd><p>The sum of finished sequences per request, in pinned memory, [BS]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput8logProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput8logProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput8logProbsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::logProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a4abf75dc398349212b15b93f843fc03c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput8logProbsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Mandatory parameters for Beam Search log-probility of generated tokens, [BS, BM, MSL], float </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput11cumLogProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput11cumLogProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput11cumLogProbsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::cumLogProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a428f7d981f0617bda37e6f64bb4f0007"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumLogProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11cumLogProbsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Sum log-probility of all generated tokens, [BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput9parentIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput9parentIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput9parentIdsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::parentIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a9f518ec59bd0df527432074008d974a6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">parentIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9parentIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Index of the beam where the previous token is, [BS, BM, MSL]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput7lengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput7lengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput7lengthsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::lengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a7425f953e704cffb2b917d475fc90c12"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput7lengthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Total sequence lengths including padding, [BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput16cacheIndirectionE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput16cacheIndirectionE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput16cacheIndirectionE"></span><span id="tensorrt_llm::runtime::DecodingOutput::cacheIndirection__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a6ee6dfc4bf8f6f7ae17e0fc8d11ea56d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheIndirection</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput16cacheIndirectionE" title="Link to this definition">#</a><br /></dt>
<dd><p>K/V indirection for next generation step, [BS, BM, MSL]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput13logProbsTiledE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput13logProbsTiledE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput13logProbsTiledE"></span><span id="tensorrt_llm::runtime::DecodingOutput::logProbsTiled__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1ac0ca8c7f11b53aa97c4caa6282853a65"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logProbsTiled</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput13logProbsTiledE" title="Link to this definition">#</a><br /></dt>
<dd><p>Buffer used to store the transpose of the logProbs, [MSL, BS, BM]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14beamHypothesesE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14beamHypothesesE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14beamHypothesesE"></span><span id="tensorrt_llm::runtime::DecodingOutput::beamHypotheses__BeamHypotheses"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a1e1c48231792a45618deff974b48b4ee"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE" title="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses"><span class="n"><span class="pre">BeamHypotheses</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">beamHypotheses</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14beamHypothesesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26speculativeDecodingOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26speculativeDecodingOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26speculativeDecodingOutputsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::speculativeDecodingOutputs__std::optional:SpeculativeDecodingOutputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a14f7bfd40b8c07bacab271148aabc2d7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE" title="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs"><span class="n"><span class="pre">SpeculativeDecodingOutputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">speculativeDecodingOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26speculativeDecodingOutputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26explicitDraftTokensBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26explicitDraftTokensBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26explicitDraftTokensBuffersE"></span><span id="tensorrt_llm::runtime::DecodingOutput::explicitDraftTokensBuffers__std::optional:ExplicitDraftTokensBuffers::Inputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a0379b1ee160c10fcdaeef426c8799a1b"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">explicitDraftTokensBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26explicitDraftTokensBuffersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput16lookaheadOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput16lookaheadOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput16lookaheadOutputsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::lookaheadOutputs__std::optional:LookaheadDecodingBuffers:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a304d012c64f4a46e8fda750824e12063"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE" title="tensorrt_llm::runtime::LookaheadDecodingBuffers"><span class="n"><span class="pre">LookaheadDecodingBuffers</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">lookaheadOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput16lookaheadOutputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput12eagleBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput12eagleBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput12eagleBuffersE"></span><span id="tensorrt_llm::runtime::DecodingOutput::eagleBuffers__std::optional:EagleBuffers::Inputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a1123acfc58f9842de75bdf8a29316b58"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput12eagleBuffersE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput17kNegativeInfinityE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput17kNegativeInfinityE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput17kNegativeInfinityE"></span><span id="tensorrt_llm::runtime::DecodingOutput::kNegativeInfinity__float"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1a1c6759c548ba677d9ea18fb8c895a6aa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kNegativeInfinity</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1e20f</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput17kNegativeInfinityE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BeamHypotheses</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5emptyERK13BufferManager">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5emptyERK13BufferManager"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5emptyERK13BufferManager"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::empty__BufferManagerCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1accd6f7899ff32a3ce977e8571e8ea4c8"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">empty</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5emptyERK13BufferManager" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7reshapeE10SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7reshapeE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7reshapeE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::reshape__SizeType32.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a481322408d5bcdd80a03cb104bca32be"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">beamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7reshapeE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7releaseEv">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7releaseEv"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7releaseEv"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::release"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a6c10ab322d436f29ce99f0f0cb4e1e1a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">release</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7releaseEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses4initERK13BufferManager11TokenIdType">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses4initERK13BufferManager11TokenIdType"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses4initERK13BufferManager11TokenIdType"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::init__BufferManagerCR.TokenIdType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a0e5faef1054124023e3a59306ce55a66"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">init</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE" title="tensorrt_llm::runtime::TokenIdType"><span class="n"><span class="pre">TokenIdType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">endId</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses4initERK13BufferManager11TokenIdType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5sliceE10SizeType3210SizeType32">
<span id="_CPPv3NK12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5sliceE10SizeType3210SizeType32"></span><span id="_CPPv2NK12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5sliceE10SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::slice__SizeType32.SizeType32C"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a50635b6fe49fb91ae5bcfe7f0c49c96e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE" title="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses"><span class="n"><span class="pre">BeamHypotheses</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchIndex</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5sliceE10SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses12outputIdsCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses12outputIdsCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses12outputIdsCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::outputIdsCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a0961829db8fc22b9d37455958b6af513"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">outputIdsCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses12outputIdsCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11logProbsCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11logProbsCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11logProbsCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::logProbsCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1ab4c51ee4b3b57de60a07e2e4e0ef1b69"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logProbsCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11logProbsCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18sequenceLengthsCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18sequenceLengthsCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18sequenceLengthsCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::sequenceLengthsCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1afd44fb8972884de1ca81f62ff7a55189"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">sequenceLengthsCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18sequenceLengthsCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses14cumLogProbsCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses14cumLogProbsCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses14cumLogProbsCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::cumLogProbsCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a7f0329d9772be169a083bd57fcfe1691"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumLogProbsCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses14cumLogProbsCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses15normedScoresCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses15normedScoresCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses15normedScoresCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::normedScoresCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a66b2659d6b50d9e9ae6fe0e8c10c9d08"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">normedScoresCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses15normedScoresCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11numBeamsCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11numBeamsCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11numBeamsCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::numBeamsCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a2369a7d4bf929356c3441fb4d9ccdc64"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">numBeamsCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11numBeamsCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18minNormedScoresCBAE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18minNormedScoresCBAE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18minNormedScoresCBAE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::minNormedScoresCBA__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a4db33cb17d3de5a6635bf03132af2633"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">minNormedScoresCBA</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18minNormedScoresCBAE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses10batchDonesE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses10batchDonesE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses10batchDonesE"></span><span id="tensorrt_llm::runtime::DecodingOutput::BeamHypotheses::batchDones__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1BeamHypotheses_1a7d6198e1a65cfea9755483162c8e139e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchDones</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses10batchDonesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs15nextDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs15nextDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs15nextDraftTokensE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::nextDraftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1a1f368e032acf0d91ee64b338ccdcf32c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs15nextDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18nextDraftTokensLenE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18nextDraftTokensLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18nextDraftTokensLenE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::nextDraftTokensLen__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1a69501f3834d3b2251b5fb8d01ec489b6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokensLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18nextDraftTokensLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18prevDraftTokensLenE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18prevDraftTokensLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18prevDraftTokensLenE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::prevDraftTokensLen__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1a52297748054f430339c310d2ac330bde"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">prevDraftTokensLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18prevDraftTokensLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs17acceptedTokensLenE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs17acceptedTokensLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs17acceptedTokensLenE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::acceptedTokensLen__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1ad1b7961868d99497e4234ad4b8f52af8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedTokensLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs17acceptedTokensLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs21acceptedLengthsCumSumE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs21acceptedLengthsCumSumE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs21acceptedLengthsCumSumE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::acceptedLengthsCumSum__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1ad830ac27b0d38fc08ae08b9a9f408058"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedLengthsCumSum</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs21acceptedLengthsCumSumE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs12pathsOffsetsE">
<span id="_CPPv3N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs12pathsOffsetsE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs12pathsOffsetsE"></span><span id="tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs::pathsOffsets__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1DecodingOutput_1_1SpeculativeDecodingOutputs_1a03210ddbe99a5780c2301d109536294b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE" title="tensorrt_llm::runtime::DecodingOutput::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">pathsOffsets</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs12pathsOffsetsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="cudastream-h">
<h2>cudaStream.h<a class="headerlink" href="#cudastream-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStreamE"></span><span id="tensorrt_llm::runtime::CudaStream"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamEji">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream10CudaStreamEji"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream10CudaStreamEji"></span><span id="tensorrt_llm::runtime::CudaStream::CudaStream__unsigned-i.i"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a2ba13a61587813c68c018a64ed2967fe"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStream</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">unsigned</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">flags</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">cudaStreamNonBlocking</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">priority</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamEji" title="Link to this definition">#</a><br /></dt>
<dd><p>Creates a new cuda stream on the current device. The stream will be destroyed in the destructor.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>flags</strong> – Flags for stream creation. See <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1CudaStream_1a9ba739eedb71b18bc4a58ff480c82a27"><span class="std std-ref">cudaStreamCreateWithFlags</span></a> for a list of valid flags that can be passed. </p></li>
<li><p><strong>priority</strong> – Priority of the stream. Lower numbers represent higher priorities. See <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1CudaStream_1a9ba739eedb71b18bc4a58ff480c82a27"><span class="std std-ref">cudaDeviceGetStreamPriorityRange</span></a> for more information about the meaningful stream priorities that can be passed. </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_tib">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_tib"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_tib"></span><span id="tensorrt_llm::runtime::CudaStream::CudaStream__cudaStream_t.i.b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a4ff09ea22fc6679e2d93b772e148d19e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStream</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">cudaStream_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">device</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ownsStream</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_tib" title="Link to this definition">#</a><br /></dt>
<dd><p>Pass an existing cuda stream to this object.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>stream</strong> – The stream to pass to this object. </p></li>
<li><p><strong>device</strong> – The device on which the stream was created. </p></li>
<li><p><strong>ownsStream</strong> – Whether this object owns the stream and destroys it in the destructor. </p></li>
</ul>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_t">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_t"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_t"></span><span id="tensorrt_llm::runtime::CudaStream::CudaStream__cudaStream_t"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a880dbbd2bd43cbf8022969311f8f7d55"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStream</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">cudaStream_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_t" title="Link to this definition">#</a><br /></dt>
<dd><p>Construct with an existing cuda stream or the default stream by passing nullptr. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream9getDeviceEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream9getDeviceEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream9getDeviceEv"></span><span id="tensorrt_llm::runtime::CudaStream::getDeviceC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a2209ae12d3b5a27740d66bec35e686c9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDevice</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream9getDeviceEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the device on which the stream was created. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream3getEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream3getEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream3getEv"></span><span id="tensorrt_llm::runtime::CudaStream::getC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a9ba739eedb71b18bc4a58ff480c82a27"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">cudaStream_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">get</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream3getEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the stream associated with this object. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream11synchronizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream11synchronizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream11synchronizeEv"></span><span id="tensorrt_llm::runtime::CudaStream::synchronizeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a3e3def9cc0e09e9724e9e68232ed2679"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">synchronize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream11synchronizeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Synchronizes the stream. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordEN9CudaEvent7pointerE">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream6recordEN9CudaEvent7pointerE"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream6recordEN9CudaEvent7pointerE"></span><span id="tensorrt_llm::runtime::CudaStream::record__CudaEvent::pointerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1acc03555968e2361dc08d28f2228a1e31"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">record</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordEN9CudaEvent7pointerE" title="Link to this definition">#</a><br /></dt>
<dd><p>Record an event on the stream. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordERK9CudaEvent">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream6recordERK9CudaEvent"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream6recordERK9CudaEvent"></span><span id="tensorrt_llm::runtime::CudaStream::record__CudaEventCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1ace6075f1266bdefe0b2033717dd8b14a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">record</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordERK9CudaEvent" title="Link to this definition">#</a><br /></dt>
<dd><p>Record an event on the stream. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitEN9CudaEvent7pointerE">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream4waitEN9CudaEvent7pointerE"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream4waitEN9CudaEvent7pointerE"></span><span id="tensorrt_llm::runtime::CudaStream::wait__CudaEvent::pointerC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a97622ff55b119c5eec1b096115462098"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wait</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE" title="tensorrt_llm::runtime::CudaEvent::pointer"><span class="n"><span class="pre">pointer</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitEN9CudaEvent7pointerE" title="Link to this definition">#</a><br /></dt>
<dd><p>Wait for an event. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitERK9CudaEvent">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream4waitERK9CudaEvent"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream4waitERK9CudaEvent"></span><span id="tensorrt_llm::runtime::CudaStream::wait__CudaEventCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a60cfbc942ee8d5f787ae66f891d91766"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wait</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">event</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitERK9CudaEvent" title="Link to this definition">#</a><br /></dt>
<dd><p>Wait for an event. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-types">Private Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream9StreamPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream9StreamPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream9StreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a42e6eb53f179659c51fbd8981383af64"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">StreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_pointer_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">cudaStream_t</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7DeleterE" title="tensorrt_llm::runtime::CudaStream::Deleter"><span class="n"><span class="pre">Deleter</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream9StreamPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7mStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7mStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7mStreamE"></span><span id="tensorrt_llm::runtime::CudaStream::mStream__StreamPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a21588e234ee724b69f92d1ef26779334"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream9StreamPtrE" title="tensorrt_llm::runtime::CudaStream::StreamPtr"><span class="n"><span class="pre">StreamPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7mStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7mDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7mDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7mDeviceE"></span><span id="tensorrt_llm::runtime::CudaStream::mDevice__i"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1a5dc4aea4b9abfa576f166ce152b675e0"></span><span class="kt"><span class="pre">int</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDevice</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7mDeviceE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7DeleterE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7DeleterE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7DeleterE"></span><span id="tensorrt_llm::runtime::CudaStream::Deleter"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1_1Deleter"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7DeleterE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEb">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEb"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEb"></span><span id="tensorrt_llm::runtime::CudaStream::Deleter::Deleter__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1_1Deleter_1aa03f43a2e696a0433dd1a1c14ce129c2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">ownsStream</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEv">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEv"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEv"></span><span id="tensorrt_llm::runtime::CudaStream::Deleter::Deleter"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1_1Deleter_1adae80ccd16ccabd753f118cdb4111a51"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Deleter</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime10CudaStream7DeleterclE12cudaStream_t">
<span id="_CPPv3NK12tensorrt_llm7runtime10CudaStream7DeleterclE12cudaStream_t"></span><span id="_CPPv2NK12tensorrt_llm7runtime10CudaStream7DeleterclE12cudaStream_t"></span><span id="tensorrt_llm::runtime::CudaStream::Deleter::call-operator__cudaStream_tCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1_1Deleter_1a077db80f4a8d30a5f2e0adde1ec6372a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">()</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">cudaStream_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">stream</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream7DeleterclE12cudaStream_t" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter11mOwnsStreamE">
<span id="_CPPv3N12tensorrt_llm7runtime10CudaStream7Deleter11mOwnsStreamE"></span><span id="_CPPv2N12tensorrt_llm7runtime10CudaStream7Deleter11mOwnsStreamE"></span><span id="tensorrt_llm::runtime::CudaStream::Deleter::mOwnsStream__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1CudaStream_1_1Deleter_1a71b80a1570697e35d7b5edb51904d9bd"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mOwnsStream</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter11mOwnsStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="eaglebuffers-h">
<h2>eagleBuffers.h<a class="headerlink" href="#eaglebuffers-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1batch__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batch_manager</span></span></span><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffersE"></span><span id="tensorrt_llm::runtime::EagleBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EagleBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a69aa6170271f65247462172a15600c88"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LlmRequestPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">LlmRequest</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a36d74a637a8d68ab93d55e9af634471a"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestVector</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE" title="tensorrt_llm::runtime::EagleBuffers::LlmRequestPtr"><span class="n"><span class="pre">LlmRequestPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers10SizeType32E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a3626c46ed5783f220200077cd9fee59f"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers7ITensorE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers7ITensorE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers7ITensorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a73d1859dd18dab0ef46c990054dc2327"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ITensor</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7ITensorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1adc9567ee5765b159fc4fc7bc38251eee"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ae97d295f67ee450da4d4c512daa21413"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorMapE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers9TensorMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers9TensorMapE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ac84f2deab51cdf8ddc998aaf4cf96e18"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="tensorrt_llm::runtime::StringPtrMap"><span class="n"><span class="pre">StringPtrMap</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers12EagleBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN8executor14DecodingConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers12EagleBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN8executor14DecodingConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers12EagleBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN8executor14DecodingConfigE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EagleBuffers__SizeType32.SizeType32.runtime::BufferManagerCR.runtime::ModelConfigCR.runtime::WorldConfigCR.executor::DecodingConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1afb0acc27f159afa3b14597a773a4d26c"></span><span class="sig-name descname"><span class="n"><span class="pre">EagleBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor14DecodingConfigE" title="tensorrt_llm::executor::DecodingConfig"><span class="n"><span class="pre">DecodingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decodingConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers12EagleBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN8executor14DecodingConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"></span><span id="tensorrt_llm::runtime::EagleBuffers::reshape__SizeType32.SizeType32.runtime::ModelConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ab7d4d9270143f5aa163a5c66b52e9f2a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVectorRKN7runtime7ITensorERK7ITensorRKN12EagleBuffers6InputsERKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE">
<span id="_CPPv3NK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVectorRKN7runtime7ITensorERK7ITensorRKN12EagleBuffers6InputsERKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="_CPPv2NK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVectorRKN7runtime7ITensorERK7ITensorRKN12EagleBuffers6InputsERKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::EagleBuffers::setFromInputs__RequestVectorCR.RequestVectorCR.runtime::ITensorCR.ITensorCR.EagleBuffers::InputsCR.runtime::BufferManagerCR.runtime::ModelConfigCR.runtime::WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a0bd6a8c780560e2099ee4667c5d57267"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFromInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE" title="tensorrt_llm::runtime::EagleBuffers::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">contextRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE" title="tensorrt_llm::runtime::EagleBuffers::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">genRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">requestTypes</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7ITensorE" title="tensorrt_llm::runtime::EagleBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decoderBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVectorRKN7runtime7ITensorERK7ITensorRKN12EagleBuffers6InputsERKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime12EagleBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE">
<span id="_CPPv3NK12tensorrt_llm7runtime12EagleBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"></span><span id="_CPPv2NK12tensorrt_llm7runtime12EagleBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::EagleBuffers::insertInputTensors__TensorMapR.TensorMapR.runtime::WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a82c9e53cd86889744a3383b89dc33f23"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">insertInputTensors</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorMapE" title="tensorrt_llm::runtime::EagleBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorMapE" title="tensorrt_llm::runtime::EagleBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">outputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime12EagleBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers12engineInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers12engineInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers12engineInputsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::engineInputs__Inputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a9659dbf9f9d1e45eddcc5c3054b680dd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers12engineInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13engineOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13engineOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13engineOutputsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::engineOutputs__tensorrt_llm::runtime::EagleBuffers::EngineOutputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1abd3e4bcd264d87334f064ceef1afd60b"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE" title="tensorrt_llm::runtime::EagleBuffers::EngineOutputs"><span class="n"><span class="pre">EngineOutputs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13engineOutputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime12EagleBuffers13setFromInputsEvRK13RequestVectorRK13RequestVector10SizeType32RK7ITensorRKN12EagleBuffers6InputsERKN7runtime11EagleModuleERKN7runtime13BufferManagerE">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVector10SizeType32RK7ITensorRKN12EagleBuffers6InputsERKN7runtime11EagleModuleERKN7runtime13BufferManagerE"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVector10SizeType32RK7ITensorRKN12EagleBuffers6InputsERKN7runtime11EagleModuleERKN7runtime13BufferManagerE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a3ad1ef640dbd717e717a1d21c1f8a501"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFromInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE" title="tensorrt_llm::runtime::EagleBuffers::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">contextRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE" title="tensorrt_llm::runtime::EagleBuffers::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">genRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSizePadded</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7ITensorE" title="tensorrt_llm::runtime::EagleBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">draftBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11EagleModuleE" title="tensorrt_llm::runtime::EagleModule"><span class="n"><span class="pre">EagleModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">eagleModule</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime12EagleBuffers13setFromInputsEvRK13RequestVectorRK13RequestVector10SizeType32RK7ITensorRKN12EagleBuffers6InputsERKN7runtime11EagleModuleERKN7runtime13BufferManagerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers26scanReduceTempStorageBytesE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers26scanReduceTempStorageBytesE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers26scanReduceTempStorageBytesE"></span><span id="tensorrt_llm::runtime::EagleBuffers::scanReduceTempStorageBytes__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a55d8b9f377af937b533ce7f3cd8b7ca7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scanReduceTempStorageBytes</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers26scanReduceTempStorageBytesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers26mDefaultPosteriorThresholdE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers26mDefaultPosteriorThresholdE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers26mDefaultPosteriorThresholdE"></span><span id="tensorrt_llm::runtime::EagleBuffers::mDefaultPosteriorThreshold__float"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a3e0a04cc62f07fb2a32d6db1a5fe2497"></span><span class="kt"><span class="pre">float</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDefaultPosteriorThreshold</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0.09f</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers26mDefaultPosteriorThresholdE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers17mDoGreedySamplingE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers17mDoGreedySamplingE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers17mDoGreedySamplingE"></span><span id="tensorrt_llm::runtime::EagleBuffers::mDoGreedySampling__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ad92b5ea646e7c2781f74cd6cfa245ccc"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDoGreedySampling</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers17mDoGreedySamplingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers21scanReduceTempStorageE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers21scanReduceTempStorageE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers21scanReduceTempStorageE"></span><span id="tensorrt_llm::runtime::EagleBuffers::scanReduceTempStorage__BufferPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ae0fefea43ce6381642eb2d7180aee23b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE" title="tensorrt_llm::runtime::EagleBuffers::BufferPtr"><span class="n"><span class="pre">BufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scanReduceTempStorage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers21scanReduceTempStorageE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers23cumSumGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers23cumSumGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers23cumSumGenerationLengthsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::cumSumGenerationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a6321256e7e048b36f6dce688a6bc1bf0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumSumGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers23cumSumGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers19maxGenerationLengthE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers19maxGenerationLengthE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers19maxGenerationLengthE"></span><span id="tensorrt_llm::runtime::EagleBuffers::maxGenerationLength__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1adc072a753d723b4359064be3c9382c1e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxGenerationLength</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers19maxGenerationLengthE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers28chunkedContextNextTokensHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers28chunkedContextNextTokensHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers28chunkedContextNextTokensHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::chunkedContextNextTokensHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1aae141d4bb117b247d0bea7252a851fea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">chunkedContextNextTokensHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers28chunkedContextNextTokensHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers18greedySamplingHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers18greedySamplingHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers18greedySamplingHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::greedySamplingHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1ab35b1bfa57573173b16f9ea4924d23fc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">greedySamplingHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers18greedySamplingHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers18posteriorAlphaHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers18posteriorAlphaHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers18posteriorAlphaHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::posteriorAlphaHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a095132fdf32dd2d709314a0f7db31c3d"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">posteriorAlphaHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers18posteriorAlphaHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers22posteriorThresholdHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers22posteriorThresholdHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers22posteriorThresholdHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::posteriorThresholdHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1a1f2a8f1a76200d0397c26328fb445f30"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">posteriorThresholdHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers22posteriorThresholdHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EngineOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs15nextDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs15nextDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs15nextDraftTokensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::nextDraftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1ac373f5004578db0db8fa9d94b07fa0ea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs15nextDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13nextDraftLensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13nextDraftLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13nextDraftLensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::nextDraftLens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1ab0e5f8a16b4bff93b94aa044e0aa353f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13nextDraftLensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14nextDraftPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14nextDraftPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14nextDraftPathsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::nextDraftPaths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1a8b7ccc53fa1f8e13984e2a5e594c7746"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14nextDraftPathsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14acceptedTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14acceptedTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14acceptedTokensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::acceptedTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1a776fdd4dc9ac5cc76f35ed41241e2bfb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14acceptedTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs12acceptedLensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs12acceptedLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs12acceptedLensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::acceptedLens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1afbdae8feb6eaffee2454743440cc6ab6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs12acceptedLensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13acceptedPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13acceptedPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13acceptedPathsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::acceptedPaths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1ae77cf5bd88086dfa071f16043007d58c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">acceptedPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13acceptedPathsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs24chunkedContextNextTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs24chunkedContextNextTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs24chunkedContextNextTokensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::EngineOutputs::chunkedContextNextTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1EngineOutputs_1a73e0fca91f622b5cddfa388b702e5062"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">chunkedContextNextTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs24chunkedContextNextTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6InputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6InputsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Inputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs6createE10SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfig">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs6createE10SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs6createE10SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfig"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::create__SizeType32.BufferManagerCR.ModelConfigCR.WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a775dde76217cdb46a119deda195851d0"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">create</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E" title="tensorrt_llm::runtime::EagleBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs6createE10SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs12temperaturesE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs12temperaturesE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs12temperaturesE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::temperatures__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1aaf2e4aef805996ca63f565953b5ec118"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">temperatures</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs12temperaturesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14posteriorAlphaE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs14posteriorAlphaE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs14posteriorAlphaE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::posteriorAlpha__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a582f5fbb06de01beb5ffb16b1494ff34"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">posteriorAlpha</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14posteriorAlphaE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18posteriorThresholdE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs18posteriorThresholdE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs18posteriorThresholdE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::posteriorThreshold__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a28a20ed76c484b1e55ee8721ed434af7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">posteriorThreshold</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18posteriorThresholdE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs16randomDataSampleE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs16randomDataSampleE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs16randomDataSampleE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::randomDataSample__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a502fd59c084cd2116b088575daed8e3f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">randomDataSample</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs16randomDataSampleE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20randomDataValidationE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs20randomDataValidationE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs20randomDataValidationE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::randomDataValidation__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1af2b06f77002c7960a14c901c0b5746e1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">randomDataValidation</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20randomDataValidationE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingTokens] or [numSequences, maxDecodingTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs11draftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs11draftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs11draftTokensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::draftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a6d4cec85660e51cb6edb3842e4bfb9d7"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs11draftTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs9draftLensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs9draftLensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs9draftLensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::draftLens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ad090ea9e7f1aa7906f125f6c82878c97"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftLens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs9draftLensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10draftPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs10draftPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs10draftPathsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::draftPaths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1afcddcc096dd0db8b8170f6759b54d4b8"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10draftPathsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14draftPathsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs14draftPathsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs14draftPathsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::draftPathsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a394bfb2132b940132ac5a348f83b32aa"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftPathsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14draftPathsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathLen] or [numSequences, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29specDecodingGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs29specDecodingGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs29specDecodingGenerationLengthsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::specDecodingGenerationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a90989ce7f1f133fe2c2bd90ce5d0ec98"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecodingGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29specDecodingGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numGenSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33specDecodingGenerationLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs33specDecodingGenerationLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs33specDecodingGenerationLengthsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::specDecodingGenerationLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a7b5bce27b39c26427043ddda02db0a1e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecodingGenerationLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33specDecodingGenerationLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numGenSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs23specDecodingPackedMasksE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs23specDecodingPackedMasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs23specDecodingPackedMasksE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::specDecodingPackedMasks__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ac19500a556ddab1fbbd2c13f3fd7df06"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecodingPackedMasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs23specDecodingPackedMasksE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27specDecodingPositionOffsetsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs27specDecodingPositionOffsetsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs27specDecodingPositionOffsetsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::specDecodingPositionOffsets__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a40ca6da2217921cca5380be65437c1a0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">specDecodingPositionOffsets</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27specDecodingPositionOffsetsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numGenSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetCtxRequestTypesHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetCtxRequestTypesHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetCtxRequestTypesHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetCtxRequestTypesHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a32fbad1915e6ef5eb5f96c2e61866f88"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetCtxRequestTypesHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetCtxRequestTypesHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetCtxContextLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetCtxContextLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetCtxContextLengthsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetCtxContextLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ae3e80bfd623785f10f74428d5ba70455"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetCtxContextLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetCtxContextLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetCtxPastKeyValueLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetCtxPastKeyValueLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetCtxPastKeyValueLengthsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetCtxPastKeyValueLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a027372839fa228e788837bd68590b9ea"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetCtxPastKeyValueLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetCtxPastKeyValueLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetGenRequestTypesHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetGenRequestTypesHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetGenRequestTypesHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetGenRequestTypesHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1adf92b9f5ff67c1c04fedf0e84e1a961b"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetGenRequestTypesHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetGenRequestTypesHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetGenContextLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetGenContextLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetGenContextLengthsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetGenContextLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a02ae021828f1174e028b64db11da1240"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetGenContextLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetGenContextLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetGenPastKeyValueLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetGenPastKeyValueLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetGenPastKeyValueLengthsHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::eagleNetGenPastKeyValueLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1abbe9ebb8982d302d6e63a3e651d351ef"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleNetGenPastKeyValueLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetGenPastKeyValueLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18inputGenTokensHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs18inputGenTokensHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs18inputGenTokensHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::inputGenTokensHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ae7c4598e87c543fdf4cd817cbcaeaae0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">inputGenTokensHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18inputGenTokensHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize * maxDecodingTokens] or [numSequences * maxDecodingTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs24chunkedContextNextTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs24chunkedContextNextTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs24chunkedContextNextTokensE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::chunkedContextNextTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1aaee773dde70e6e6631a8b4848e0e5efe"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">chunkedContextNextTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs24chunkedContextNextTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15useSpecDecodingE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs15useSpecDecodingE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs15useSpecDecodingE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::useSpecDecoding__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ae0fc780a1be8f1844c241ec73f762bcb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useSpecDecoding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15useSpecDecodingE" title="Link to this definition">#</a><br /></dt>
<dd><p>[1] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18useDynamicTreeHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs18useDynamicTreeHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs18useDynamicTreeHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::useDynamicTreeHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1acda2031f996b58ffe592de1a2d128e98"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useDynamicTreeHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18useDynamicTreeHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[1] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22dynamicTreeMaxTopKHostE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs22dynamicTreeMaxTopKHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs22dynamicTreeMaxTopKHostE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::dynamicTreeMaxTopKHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a3afac314a4d8cdba08abc3f029663f6c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">dynamicTreeMaxTopKHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22dynamicTreeMaxTopKHostE" title="Link to this definition">#</a><br /></dt>
<dd><p>[1] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10prevScoresE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs10prevScoresE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs10prevScoresE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::prevScores__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1ac542e8c82af5dacaa889d97c33408332"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">prevScores</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10prevScoresE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20currentExpandIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs20currentExpandIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs20currentExpandIndicesE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::currentExpandIndices__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a4e384e257c4aa32b74c6340bda567e9f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">currentExpandIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20currentExpandIndicesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingDraftTokens] or [numSequences, maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15allLayersScoresE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs15allLayersScoresE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs15allLayersScoresE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::allLayersScores__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a943a377aa05ce47eeed5ec62cf9773bc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allLayersScores</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15allLayersScoresE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22allLayersDraftTokenIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs22allLayersDraftTokenIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs22allLayersDraftTokenIdsE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::allLayersDraftTokenIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a8f370650031447cd7fb716ca31f44da6"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allLayersDraftTokenIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22allLayersDraftTokenIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33allLayersDraftTokenIdsPredecessorE">
<span id="_CPPv3N12tensorrt_llm7runtime12EagleBuffers6Inputs33allLayersDraftTokenIdsPredecessorE"></span><span id="_CPPv2N12tensorrt_llm7runtime12EagleBuffers6Inputs33allLayersDraftTokenIdsPredecessorE"></span><span id="tensorrt_llm::runtime::EagleBuffers::Inputs::allLayersDraftTokenIdsPredecessor__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1EagleBuffers_1_1Inputs_1a899645bdaeb2178b36e2840bd5e27082"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE" title="tensorrt_llm::runtime::EagleBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">allLayersDraftTokenIdsPredecessor</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33allLayersDraftTokenIdsPredecessorE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] or [numSequences, numEagleLayers, maxDecodingDraftTokens * maxDecodingDraftTokens] </p>
</dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="igptdecoderbatched-h">
<h2>iGptDecoderBatched.h<a class="headerlink" href="#igptdecoderbatched-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1batch__manager"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batch_manager</span></span></span><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatchedE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatchedE"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IGptDecoderBatched</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;iGptDecoderBatched.h&gt;</em></div>
<p>GPT decoder class with support for in-flight batching. </p>
<p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1GptDecoderBatched"><span class="std std-ref">tensorrt_llm::runtime::GptDecoderBatched</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13CudaStreamPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched13CudaStreamPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched13CudaStreamPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a2a27a496ac11aeb918dede4d513568aa"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">CudaStreamPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13CudaStreamPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a41f59ec19ac27bdc5cd92778f3d8d2a9"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">LlmRequestPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm13batch_managerE" title="tensorrt_llm::batch_manager"><span class="n"><span class="pre">batch_manager</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">LlmRequest</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1ad580b2d9549986c709a235dc161f21c0"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RequestVector</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE" title="tensorrt_llm::runtime::IGptDecoderBatched::LlmRequestPtr"><span class="n"><span class="pre">LlmRequestPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a99e95a44eda53ca55f2e7efeba372229"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::setup__executor::DecodingModeCR.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.SizeType32.nvinfer1::DataType.ModelConfigCR.WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a78998c48fc4fd319ea2696de8b0bfb1a"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setup</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor12DecodingModeE" title="tensorrt_llm::executor::DecodingMode"><span class="n"><span class="pre">DecodingMode</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">mode</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxAttentionWindow</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">sinkTokenLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxSequenceLength</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxTokensPerStep</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dtype</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig" title="Link to this definition">#</a><br /></dt>
<dd><p>Setup the decoder before calling <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a57f2c8ee8a7a6cdb36d93c40ff04d7cc"><span class="std std-ref"><span class="pre">forward()</span></span></a></code></p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::disableLookahead__RequestVectorCR.TensorPtrCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a313c1c30cfc0b827ac8b74835550e4aa"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">disableLookahead</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE" title="tensorrt_llm::runtime::IGptDecoderBatched::RequestVector"><span class="n"><span class="pre">RequestVector</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">genRequests</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE" title="tensorrt_llm::runtime::IGptDecoderBatched::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">batchSlots</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr" title="Link to this definition">#</a><br /></dt>
<dd><p>Disable Lookahead decoding. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::forwardAsync__decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a90191ce0e50206fb5cb1f7fb3aa2acda"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forwardAsync</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Run one step for all requests without blocking the host process and return the token for synchronization. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::forward__decoder_batch::OutputR.decoder_batch::InputCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a57f2c8ee8a7a6cdb36d93c40ff04d7cc"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">forward</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="tensorrt_llm::runtime::decoder_batch::Output"><span class="n"><span class="pre">Output</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE" title="tensorrt_llm::runtime::decoder_batch"><span class="n"><span class="pre">decoder_batch</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="tensorrt_llm::runtime::decoder_batch::Input"><span class="n"><span class="pre">Input</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">input</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><p>Run one step for all requests and wait for completion on the host. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime18IGptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb">
<span id="_CPPv3NK12tensorrt_llm7runtime18IGptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"></span><span id="_CPPv2NK12tensorrt_llm7runtime18IGptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::finalize__decoder::DecoderStateCR.SizeType32.SamplingConfigCR.bC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a654f7a2460e7e69fb32d96cbb9546b54"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE" title="tensorrt_llm::runtime::CudaEvent"><span class="n"><span class="pre">CudaEvent</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">finalize</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoderE" title="tensorrt_llm::runtime::decoder"><span class="n"><span class="pre">decoder</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE" title="tensorrt_llm::runtime::decoder::DecoderState"><span class="n"><span class="pre">DecoderState</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decoderState</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">batchSlot</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE" title="tensorrt_llm::runtime::SamplingConfig"><span class="n"><span class="pre">SamplingConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">samplingConfig</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">streaming</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime18IGptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb" title="Link to this definition">#</a><br /></dt>
<dd><p>Gather final beam search results for request <code class="docutils literal notranslate"><span class="pre">batchIdx</span></code>. Result will only be available after event returned. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-protected-functions">Protected Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched18IGptDecoderBatchedEv">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatched18IGptDecoderBatchedEv"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatched18IGptDecoderBatchedEv"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::IGptDecoderBatched"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1ab7ee2bc18a2287c62d86ebf02f2c6f68"></span><span class="sig-name descname"><span class="n"><span class="pre">IGptDecoderBatched</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched18IGptDecoderBatchedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime18IGptDecoderBatchedD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime18IGptDecoderBatchedD0Ev"></span><span id="tensorrt_llm::runtime::IGptDecoderBatched::~IGptDecoderBatched"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IGptDecoderBatched_1a90e634d85109a220dff1b2567e1d8f7f"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~IGptDecoderBatched</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime_1_1decoder"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">decoder</span></span></span><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime_1_1decoder__batch"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">decoder_batch</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5InputE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5InputE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Input</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1aa8152e055448762bd78ad70f53eda8ba"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ac182ee5aa51be63dfe20586ecaf40043"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorINSt6vectorI14TensorConstPtrEEEE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorINSt6vectorI14TensorConstPtrEEEE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorINSt6vectorI14TensorConstPtrEEEE10SizeType32"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::Input__std::vector:std::vector:TensorConstPtr::CR.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a041ff27c7d9d44312e45c2bbefcfb58d"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Input</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">logits</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDecoderSteps</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorINSt6vectorI14TensorConstPtrEEEE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorI14TensorConstPtrEE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorI14TensorConstPtrEE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorI14TensorConstPtrEE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::Input__std::vector:TensorConstPtr:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1aba21ef996e9e6fc1aca5bcc09fcd55ad"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Input</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">logits</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorI14TensorConstPtrEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input6logitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input6logitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input6logitsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::logits__std::vector:std::vector:TensorConstPtr::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a545f3bcadd377eea1d80f1271e066ffd"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorConstPtr"><span class="n"><span class="pre">TensorConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">logits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input6logitsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxDecoderSteps][batchSize][1, beamWidth, vocabSizePadded], on gpu </p>
<p>Mandatory parameters Logits </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15maxDecoderStepsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input15maxDecoderStepsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input15maxDecoderStepsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::maxDecoderSteps__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ab42c96a0c6f908ff046599c7233aa8f1"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxDecoderSteps</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15maxDecoderStepsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Maximum number of decoding tokens of active slots. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input10batchSlotsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input10batchSlotsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input10batchSlotsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::batchSlots__std::vector:TensorPtr:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a06f6d5749efcad06630072eb17f1a6d9"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchSlots</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input10batchSlotsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Batch of active decoder slots, sorted by slots, [maxDecoderSteps][batchSize]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input22batchSlotsRequestOrderE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input22batchSlotsRequestOrderE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input22batchSlotsRequestOrderE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::batchSlotsRequestOrder__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1acf51193dc7d88e0754b2fde747f5ff12"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">batchSlotsRequestOrder</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input22batchSlotsRequestOrderE" title="Link to this definition">#</a><br /></dt>
<dd><p>Filled with slots in request order, [batchSize]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input16cacheIndirectionE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input16cacheIndirectionE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input16cacheIndirectionE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::cacheIndirection__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ad1b93f5e2145ae272a72811679f7673a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheIndirection</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input16cacheIndirectionE" title="Link to this definition">#</a><br /></dt>
<dd><p>For Beam Search Indices into KV cache of different rays within one beam, [maxBatchSize, maxBeamWidth, maxSeqLen], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15generationStepsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input15generationStepsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input15generationStepsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::generationSteps__std::vector:SizeType32:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ab585fbd53121a6094355bdc7cc5c195d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationSteps</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15generationStepsE" title="Link to this definition">#</a><br /></dt>
<dd><p>The generation step of each request (for Variable-Beam-Width-Search), [batchSize]. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input20predictedDraftLogitsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input20predictedDraftLogitsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input20predictedDraftLogitsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::predictedDraftLogits__std::vector:std::vector:TensorPtr::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a6692df8763ceda9c12f94f4553d32ed1"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Input::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">predictedDraftLogits</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input20predictedDraftLogitsE" title="Link to this definition">#</a><br /></dt>
<dd><p>For speculative decoding Logits of draft [maxBatchSize][maxAcceptedDraftTokensPerStep][maxDraftTokens + 1, vocabSizePadded] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input25explicitDraftTokensInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input25explicitDraftTokensInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input25explicitDraftTokensInputsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::explicitDraftTokensInputs__std::optional:ExplicitDraftTokensBuffers::EngineOutputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ae58255408d173c03cb6efb2a988b8479"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs"><span class="n"><span class="pre">EngineOutputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">explicitDraftTokensInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input25explicitDraftTokensInputsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Explicit draft tokens data. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input29explicitDraftTokensLastInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input29explicitDraftTokensLastInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input29explicitDraftTokensLastInputsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::explicitDraftTokensLastInputs__std::optional:ExplicitDraftTokensBuffers::EngineInputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a41a98f4e5c7f05757a82300ede64ac4d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs"><span class="n"><span class="pre">EngineInputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">explicitDraftTokensLastInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input29explicitDraftTokensLastInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input11eagleInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input11eagleInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input11eagleInputsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::eagleInputs__std::optional:EagleBuffers::EngineOutputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1ab53513121aa63b63d84569172619b207"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE" title="tensorrt_llm::runtime::EagleBuffers::EngineOutputs"><span class="n"><span class="pre">EngineOutputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input11eagleInputsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Eagle data. </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15eagleLastInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch5Input15eagleLastInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch5Input15eagleLastInputsE"></span><span id="tensorrt_llm::runtime::decoder_batch::Input::eagleLastInputs__std::optional:EagleBuffers::Inputs:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Input_1a68e11bed66f250eedc5e74d45973fcaf"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE" title="tensorrt_llm::runtime::EagleBuffers"><span class="n"><span class="pre">EagleBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE" title="tensorrt_llm::runtime::EagleBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">eagleLastInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15eagleLastInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch6OutputE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch6OutputE"></span><span id="tensorrt_llm::runtime::decoder_batch::Output"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Output"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Output</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Output_1ab1a4d43f1e25e7ffc107edf438b75ff8"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output6OutputEv">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch6Output6OutputEv"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch6Output6OutputEv"></span><span id="tensorrt_llm::runtime::decoder_batch::Output::Output"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Output_1a11aaaeb3c20e5f93d815b2b6c86f345f"></span><span class="sig-name descname"><span class="n"><span class="pre">Output</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output6OutputEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output16cacheIndirectionE">
<span id="_CPPv3N12tensorrt_llm7runtime13decoder_batch6Output16cacheIndirectionE"></span><span id="_CPPv2N12tensorrt_llm7runtime13decoder_batch6Output16cacheIndirectionE"></span><span id="tensorrt_llm::runtime::decoder_batch::Output::cacheIndirection__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1decoder__batch_1_1Output_1ab579f71d5d5639305bb67ecf1a58ba55"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE" title="tensorrt_llm::runtime::decoder_batch::Output::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cacheIndirection</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output16cacheIndirectionE" title="Link to this definition">#</a><br /></dt>
<dd><p>parameters for beam search, [batchSize, maxBeamWidth, maxSeqLen], on gpu </p>
</dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="speculativedecodingmodule-h">
<h2>speculativeDecodingModule.h<a class="headerlink" href="#speculativedecodingmodule-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModuleE"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModuleE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="Link to this definition">#</a><br /></dt>
<dd><p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1EagleModule"><span class="std std-ref">tensorrt_llm::runtime::EagleModule</span></a>, <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1LookaheadModule"><span class="std std-ref">tensorrt_llm::runtime::LookaheadModule</span></a>, <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1MedusaModule"><span class="std std-ref">tensorrt_llm::runtime::MedusaModule</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleE10SizeType3210SizeType3210SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleE10SizeType3210SizeType3210SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleE10SizeType3210SizeType3210SizeType32"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::SpeculativeDecodingModule__SizeType32.SizeType32.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1aabf35face1ea9413d8b378bd70a33280"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftPathLen</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDecodingDraftTokens</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumPaths</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleE10SizeType3210SizeType3210SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleEv">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleEv"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::SpeculativeDecodingModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a1af69740c57c1f1a8e7590caf30b1a44"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModuleD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModuleD0Ev"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::~SpeculativeDecodingModule"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a59776b8ad4a90d0a906bf00c619554da"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~SpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::SpeculativeDecodingModule__SpeculativeDecodingModuleCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a0793cd049c1b7502c683ad8c8da48008"></span><span class="sig-name descname"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule" title="tensorrt_llm::runtime::SpeculativeDecodingModule::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleaSERK25SpeculativeDecodingModule">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModuleaSERK25SpeculativeDecodingModule"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModuleaSERK25SpeculativeDecodingModule"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::assign-operator__SpeculativeDecodingModuleCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1ad166641565c8f32ebf6afdc788b546a3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE" title="tensorrt_llm::runtime::SpeculativeDecodingModule"><span class="n"><span class="pre">SpeculativeDecodingModule</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">o</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleaSERK25SpeculativeDecodingModule" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule18getMaxDraftPathLenEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule18getMaxDraftPathLenEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule18getMaxDraftPathLenEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getMaxDraftPathLenC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a92a91b11ffd70a0fd2698cd6f1c96a69"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDraftPathLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule18getMaxDraftPathLenEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>max number of draft tokens that can be accepted by one step of the decoder </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule13getMaxPathLenEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule13getMaxPathLenEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule13getMaxPathLenEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getMaxPathLenC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1ab4b8c33566b684613b230b0b740e0ded"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxPathLen</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule13getMaxPathLenEv" title="Link to this definition">#</a><br /></dt>
<dd><p>
one more than draft path len for prediction from primary head </p>
<dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>max number of tokens that a request can grow in one step of the decoder</p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule25getMaxDecodingDraftTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule25getMaxDecodingDraftTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule25getMaxDecodingDraftTokensEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getMaxDecodingDraftTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a3b693afe821b966e8312236e16f527f2"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingDraftTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule25getMaxDecodingDraftTokensEv" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>max number of draft tokens processed by one step of the decoder </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule20getMaxDecodingTokensEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule20getMaxDecodingTokensEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule20getMaxDecodingTokensEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getMaxDecodingTokensC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a6fe241359c3e7969a7b506f4a9b431e3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxDecodingTokens</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule20getMaxDecodingTokensEv" title="Link to this definition">#</a><br /></dt>
<dd><p>
one more than decoding draft tokens for prediction from primary head </p>
<dl class="field-list simple">
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>max number of tokens processed by one step of the decoder</p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule17getNumPackedMasksEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule17getNumPackedMasksEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule17getNumPackedMasksEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getNumPackedMasksC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a78cb7cbc32f4f2cb9a2864c65c3c0d0a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getNumPackedMasks</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule17getNumPackedMasksEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule14getMaxNumPathsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime25SpeculativeDecodingModule14getMaxNumPathsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime25SpeculativeDecodingModule14getMaxNumPathsEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::getMaxNumPathsC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a462630e52247e02e3c01c98a23d1f705"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMaxNumPaths</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule14getMaxNumPathsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule17setMaxDraftTokensE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule17setMaxDraftTokensE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule17setMaxDraftTokensE10SizeType32"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::setMaxDraftTokens__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a6bfb797bb4ba7bfba00fbbdec168e983"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxDraftTokens</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftTokens</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule17setMaxDraftTokensE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18setMaxDraftPathLenE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule18setMaxDraftPathLenE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule18setMaxDraftPathLenE10SizeType32"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::setMaxDraftPathLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a6f550a2b2d37b70fd653b5738da40be3"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxDraftPathLen</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxDraftPathLen</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18setMaxDraftPathLenE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule14setMaxNumPathsE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule14setMaxNumPathsE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule14setMaxNumPathsE10SizeType32"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::setMaxNumPaths__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a84e469ace7d51c3fe1d38d02a679480f"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setMaxNumPaths</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumPaths</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule14setMaxNumPathsE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule21computeNumPackedMasksEv">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule21computeNumPackedMasksEv"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule21computeNumPackedMasksEv"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::computeNumPackedMasks"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1af7f830a6266ee46c0b91414b70c36c6b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">computeNumPackedMasks</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule21computeNumPackedMasksEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule16mMaxDraftPathLenE">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule16mMaxDraftPathLenE"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule16mMaxDraftPathLenE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::mMaxDraftPathLen__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a8bf6280178a97c5e534ac182bb4f301f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxDraftPathLen</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule16mMaxDraftPathLenE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule23mMaxDecodingDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule23mMaxDecodingDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule23mMaxDecodingDraftTokensE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::mMaxDecodingDraftTokens__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1af863071adbd70ecffa6659a361c200de"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxDecodingDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule23mMaxDecodingDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule12mMaxNumPathsE">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule12mMaxNumPathsE"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule12mMaxNumPathsE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::mMaxNumPaths__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a00d82632302e77fd010e983272e65410"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxNumPaths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule12mMaxNumPathsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18mMaxNumPackedMasksE">
<span id="_CPPv3N12tensorrt_llm7runtime25SpeculativeDecodingModule18mMaxNumPackedMasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime25SpeculativeDecodingModule18mMaxNumPackedMasksE"></span><span id="tensorrt_llm::runtime::SpeculativeDecodingModule::mMaxNumPackedMasks__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1SpeculativeDecodingModule_1a0229e91e73bbb60aa933cc5c37e1deab"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mMaxNumPackedMasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18mMaxNumPackedMasksE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="explicitdrafttokensbuffers-h">
<h2>explicitDraftTokensBuffers.h<a class="headerlink" href="#explicitdrafttokensbuffers-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1ae16955c34d34bc7ea599673e766f1575"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SizeType32</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a8bc7539f7377aee16c532859d7555c82"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ITensor</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a8eaca3cd772329f8f8e89643f031b0dc"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1ab9edc81aa29738b47db692fdd0b976ae"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a707525e8e166f8c9777616891ba80177"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="tensorrt_llm::runtime::StringPtrMap"><span class="n"><span class="pre">StringPtrMap</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers26ExplicitDraftTokensBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers26ExplicitDraftTokensBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers26ExplicitDraftTokensBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::ExplicitDraftTokensBuffers__SizeType32.SizeType32.runtime::BufferManagerCR.runtime::ModelConfigCR.runtime::WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a56fa78b032b841d09e5595b634998bbf"></span><span class="sig-name descname"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBatchSize</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxBeamWidth</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers26ExplicitDraftTokensBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::reshape__SizeType32.SizeType32.runtime::ModelConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a9d1e2809ab99e9bb1868f0bc6945355b"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType32RKN7runtime7ITensorERK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN7runtime13BufferManagerERKN7runtime10CudaStreamE">
<span id="_CPPv3NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType32RKN7runtime7ITensorERK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN7runtime13BufferManagerERKN7runtime10CudaStreamE"></span><span id="_CPPv2NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType32RKN7runtime7ITensorERK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN7runtime13BufferManagerERKN7runtime10CudaStreamE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::setFromInputs__SizeType32.SizeType32.runtime::ITensorCR.ITensorCR.ExplicitDraftTokensBuffers::InputsCR.ITensorCR.runtime::ModelConfigCR.runtime::WorldConfigCR.runtime::BufferManagerCR.runtime::CudaStreamCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1ac2560a64dc8f580b887f200d3af7a8f4"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFromInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">requestTypes</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">decoderBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">contextPositionIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType32RKN7runtime7ITensorERK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN7runtime13BufferManagerERKN7runtime10CudaStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE">
<span id="_CPPv3NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"></span><span id="_CPPv2NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::insertInputTensors__TensorMapR.TensorMapR.runtime::WorldConfigCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1af00050bf6dc0ba3b3051417b8cdc2e0a"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">insertInputTensors</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">inputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorMap"><span class="n"><span class="pre">TensorMap</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">outputBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12engineInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12engineInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12engineInputsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::engineInputs__tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1ab9694adea98429287d1f04738f715e8b"></span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs"><span class="n"><span class="pre">EngineInputs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineInputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12engineInputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13engineOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13engineOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13engineOutputsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::engineOutputs__tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a89f87ae62483ce734c58a1cc254f7dba"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs"><span class="n"><span class="pre">EngineOutputs</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">engineOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13engineOutputsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers20scanTempStorageBytesE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers20scanTempStorageBytesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers20scanTempStorageBytesE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::scanTempStorageBytes__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a4221d1ab92b52026d1228577030efb2c"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scanTempStorageBytes</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers20scanTempStorageBytesE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers15scanTempStorageE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers15scanTempStorageE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers15scanTempStorageE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::scanTempStorage__BufferPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a8b0159a49cf6861dabc340a37defea9c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::BufferPtr"><span class="n"><span class="pre">BufferPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">scanTempStorage</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers15scanTempStorageE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers23cumSumGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers23cumSumGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers23cumSumGenerationLengthsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::cumSumGenerationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1acdfeec1653fe23e61364380a7fbf7e01"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">cumSumGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers23cumSumGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-functions">Private Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsEv10SizeType3210SizeType3210SizeType32RK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime25ExplicitDraftTokensModuleERKN7runtime10CudaStreamE">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType3210SizeType32RK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime25ExplicitDraftTokensModuleERKN7runtime10CudaStreamE"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType3210SizeType32RK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime25ExplicitDraftTokensModuleERKN7runtime10CudaStreamE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1a0d5e9b3f821cac5d410ea942fedf0387"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setFromInputs</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numCtxSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">numGenSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">vocabSizePadded</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">seqSlots</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">draftBuffers</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">contextPositionIds</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ExplicitDraftTokensModule</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">explicitDraftTokensModule</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE" title="tensorrt_llm::runtime::CudaStream"><span class="n"><span class="pre">CudaStream</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">stream</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsEv10SizeType3210SizeType3210SizeType32RK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime25ExplicitDraftTokensModuleERKN7runtime10CudaStreamE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineInputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EngineInputs</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers"><span class="n"><span class="pre">ExplicitDraftTokensBuffers</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"><span class="n"><span class="pre">Inputs</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs18requestTypesDeviceE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs18requestTypesDeviceE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs18requestTypesDeviceE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs::requestTypesDevice__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineInputs_1a2d8a7bc0941d924fcb3f010cacaccd41"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">requestTypesDevice</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs18requestTypesDeviceE" title="Link to this definition">#</a><br /></dt>
<dd><p>[numSequences], on gpu </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs15positionOffsetsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs15positionOffsetsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs15positionOffsetsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs::positionOffsets__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineInputs_1a8c983b75148ff16930a4662a41f7f53a"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionOffsets</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs15positionOffsetsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[numGenSequences] </p>
</dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">EngineOutputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs21nextGenerationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs21nextGenerationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs21nextGenerationLengthsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextGenerationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a6b4e603a85134a54aa2b450efef3605c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextGenerationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs21nextGenerationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs19nextPositionOffsetsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs19nextPositionOffsetsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs19nextPositionOffsetsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextPositionOffsets__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a873b44001f468d29c7a54009692b5de4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextPositionOffsets</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs19nextPositionOffsetsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs5masksE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs5masksE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs5masksE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::masks__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1affd515e8d2260e3f66e5f7e05868cc7e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">masks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs5masksE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxDecodingTokens, maxDecodingTokens], bool </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15nextDraftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15nextDraftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15nextDraftTokensE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextDraftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a7847820053d967ae770aa92fda4cd3c4"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15nextDraftTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs16nextDraftIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs16nextDraftIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs16nextDraftIndicesE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextDraftIndices__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a789bcc16137e2159d552c4e01057690e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs16nextDraftIndicesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextDraftProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextDraftProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextDraftProbsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextDraftProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a332e0b1cdf83ce26692e3261e3d789ef"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextDraftProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextDraftProbsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize, maxNumPaths, maxDraftPathLen, vocabSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextFlatTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextFlatTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextFlatTokensE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::nextFlatTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a14791c38f4511bd70a0352cbe1593205"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nextFlatTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextFlatTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize * maxDecodingTokens] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathLengthsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::bestPathLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a0228355bd26026e839d1d7ebad0b9850"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bestPathLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathLengthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathIndicesE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::bestPathIndices__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a2495fd1a5ae33ab9c5842393e76df22e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">bestPathIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathIndicesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs11maxGenTokenE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs11maxGenTokenE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs11maxGenTokenE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::maxGenToken__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1a6485ad2ce4769f39046e04c07e6a81d3"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxGenToken</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs11maxGenTokenE" title="Link to this definition">#</a><br /></dt>
<dd><p>[1] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs13totalGenTokenE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs13totalGenTokenE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs13totalGenTokenE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::totalGenToken__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1aded5114059314a9d25b91878a33b1a27"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">totalGenToken</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs13totalGenTokenE" title="Link to this definition">#</a><br /></dt>
<dd><p>[1] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs17packedPositionIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs17packedPositionIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs17packedPositionIdsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs::packedPositionIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineOutputs_1ad47080e6b28ea054c28fcac3a7fb0d7e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedPositionIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs17packedPositionIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[batchSize * maxDecodingTokens] </p>
</dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Inputs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE" title="Link to this definition">#</a><br /></dt>
<dd><p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1EngineInputs"><span class="std std-ref">tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs6createE10SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs6createE10SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs6createE10SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::create__SizeType32.runtime::BufferManagerCR.runtime::ModelConfigCR.runtime::WorldConfigCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1aa0607dfb75ce02435b0048fa3f136973"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">create</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">maxNumSequences</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE" title="tensorrt_llm::runtime::BufferManager"><span class="n"><span class="pre">BufferManager</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">manager</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE" title="tensorrt_llm::runtime::ModelConfig"><span class="n"><span class="pre">ModelConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">modelConfig</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE" title="tensorrt_llm::runtime::WorldConfig"><span class="n"><span class="pre">WorldConfig</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">worldConfig</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs6createE10SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12temperaturesE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12temperaturesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12temperaturesE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::temperatures__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1ae5e91bdb5b475a4b997f442ac337407f"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">temperatures</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12temperaturesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15positionIdsBaseE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15positionIdsBaseE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15positionIdsBaseE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::positionIdsBase__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a95c39b0d1bdf3f2dc1e73ec7a4c609dd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIdsBase</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15positionIdsBaseE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs17generationLengthsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs17generationLengthsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs17generationLengthsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::generationLengths__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1ae57a628c81b0b92c11ea515159ebbf77"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengths</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs17generationLengthsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numGenSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16randomDataSampleE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16randomDataSampleE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16randomDataSampleE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::randomDataSample__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1adbee5ac8d0c326af6066aee363de73cc"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">randomDataSample</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16randomDataSampleE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs20randomDataValidationE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs20randomDataValidationE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs20randomDataValidationE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::randomDataValidation__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1ab82f2a2dd80371b340cfe08c672600bb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">randomDataValidation</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs20randomDataValidationE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathDraftLen] or [numGenSequences, maxNumPaths, maxPathDraftLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11draftTokensE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11draftTokensE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11draftTokensE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::draftTokens__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a083b6d57fd07a2b358322290753e13c0"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftTokens</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11draftTokensE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12draftIndicesE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12draftIndicesE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12draftIndicesE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::draftIndices__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1ae73718aeff8a57a933308d684abaa9eb"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftIndices</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12draftIndicesE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathLen] or [numGenSequences, maxNumPaths, maxPathLen] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs10draftProbsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs10draftProbsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs10draftProbsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::draftProbs__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1aeabf159b722e68e8171c3dc195a5acca"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">draftProbs</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs10draftProbsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxNumPaths, maxPathDraftLen, vocabSize] or [numGenSequences, maxNumPaths, maxPathDraftLen, vocabSize] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11packedMasksE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11packedMasksE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11packedMasksE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::packedMasks__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a58685fde1bf5e57bcf3f1a81fb87a550"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">packedMasks</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11packedMasksE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize, maxDecodingTokens, ceil(maxDecodingTokens / 32)] or [numGenSequences, maxDecodingTokens, ceil(maxDecodingTokens / 32)] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11positionIdsE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11positionIdsE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11positionIdsE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::positionIds__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a89285e502e317080a813cea06dc7cfef"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">positionIds</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11positionIdsE" title="Link to this definition">#</a><br /></dt>
<dd><p>[maxBatchSize] or [numGenSequences] </p>
</dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16maxGenLengthHostE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16maxGenLengthHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16maxGenLengthHostE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::maxGenLengthHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a895e7fb9779d0440dfce06f2269b01fd"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">maxGenLengthHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16maxGenLengthHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs21generationLengthsHostE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs21generationLengthsHostE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs21generationLengthsHostE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::generationLengthsHost__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1a04c3c6ce76901450a397d18f0e082bed"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">generationLengthsHost</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs21generationLengthsHostE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15useSpecDecodingE">
<span id="_CPPv3N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15useSpecDecodingE"></span><span id="_CPPv2N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15useSpecDecodingE"></span><span id="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs::useSpecDecoding__TensorPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ExplicitDraftTokensBuffers_1_1Inputs_1ad79f539b3d0e47dcdf9f9554a0bcf13c"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE" title="tensorrt_llm::runtime::ExplicitDraftTokensBuffers::TensorPtr"><span class="n"><span class="pre">TensorPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">useSpecDecoding</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15useSpecDecodingE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="rawengine-h">
<h2>rawEngine.h<a class="headerlink" href="#rawengine-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngineE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngineE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngineE"></span><span id="tensorrt_llm::runtime::RawEngine"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RawEngine</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngineE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp enum">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine4TypeE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine4TypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine4TypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3c8f3acf0c858144e0987346247af075"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Type</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4TypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine4Type8FilePathE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine4Type8FilePathE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine4Type8FilePathE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3c8f3acf0c858144e0987346247af075a05f8d2398fcf614f8784248055f32e17"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">FilePath</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type8FilePathE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine4Type15AddressWithSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine4Type15AddressWithSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine4Type15AddressWithSizeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3c8f3acf0c858144e0987346247af075a05dee9767a15cf70383d2faf6974afe9"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">AddressWithSize</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type15AddressWithSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine4Type10HostMemoryE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine4Type10HostMemoryE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine4Type10HostMemoryE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3c8f3acf0c858144e0987346247af075aff64ba05059375611eb50ec057d3996f"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">HostMemory</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type10HostMemoryE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineENSt10filesystem4pathE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine9RawEngineENSt10filesystem4pathE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine9RawEngineENSt10filesystem4pathE"></span><span id="tensorrt_llm::runtime::RawEngine::RawEngine__std::filesystem::path"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1adea0c457ec4dcaef069c6d9cb218014b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RawEngine</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enginePath</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineENSt10filesystem4pathE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKvNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine9RawEngineEPKvNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine9RawEngineEPKvNSt6size_tE"></span><span id="tensorrt_llm::runtime::RawEngine::RawEngine__voidCP.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3052697810402d1db99e6c5a4da1bb5a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RawEngine</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">engineAddr</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">engineSize</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKvNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKN8nvinfer111IHostMemoryE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine9RawEngineEPKN8nvinfer111IHostMemoryE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine9RawEngineEPKN8nvinfer111IHostMemoryE"></span><span id="tensorrt_llm::runtime::RawEngine::RawEngine__nvinfer1::IHostMemoryCP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1abdaa133b12d2a8a8ff00ee6ac81b6d67"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">RawEngine</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">IHostMemory</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">engineBuffer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKN8nvinfer111IHostMemoryE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine7getTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine7getTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine7getTypeEv"></span><span id="tensorrt_llm::runtime::RawEngine::getTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a77dc57a1fbc1a105a2cf206e030ad1bf"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4TypeE" title="tensorrt_llm::runtime::RawEngine::Type"><span class="n"><span class="pre">Type</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine7getPathEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine7getPathEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine7getPathEv"></span><span id="tensorrt_llm::runtime::RawEngine::getPathC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a737a7c9d9876ad0cc3b21cf5926a5787"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPath</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getPathEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine10getPathOptEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine10getPathOptEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine10getPathOptEv"></span><span id="tensorrt_llm::runtime::RawEngine::getPathOptC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a420b0935cefb09c44966918cec37523b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getPathOpt</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine10getPathOptEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine7setPathENSt10filesystem4pathE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine7setPathENSt10filesystem4pathE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine7setPathENSt10filesystem4pathE"></span><span id="tensorrt_llm::runtime::RawEngine::setPath__std::filesystem::path"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a29fa3156e7ef0efc344cef39be7536f9"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setPath</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">enginePath</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine7setPathENSt10filesystem4pathE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine23getManagedWeightsMapOptEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine23getManagedWeightsMapOptEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine23getManagedWeightsMapOptEv"></span><span id="tensorrt_llm::runtime::RawEngine::getManagedWeightsMapOptC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a49ff9f55b13ee8dcb8b240b39f9fe4ab"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getManagedWeightsMapOpt</span></span></span><span class="sig-paren">(</span>
<dl>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine23getManagedWeightsMapOptEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine20setManagedWeightsMapENSt3mapINSt6stringEN12tensorrt_llm8executor6TensorEEE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine20setManagedWeightsMapENSt3mapINSt6stringEN12tensorrt_llm8executor6TensorEEE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine20setManagedWeightsMapENSt3mapINSt6stringEN12tensorrt_llm8executor6TensorEEE"></span><span id="tensorrt_llm::runtime::RawEngine::setManagedWeightsMap__std::map:ss.tensorrt_llm::executor::Tensor:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1ab38ef50e4e4107e0fa3344a33787f210"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">setManagedWeightsMap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">managedWeightsMap</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine20setManagedWeightsMapENSt3mapINSt6stringEN12tensorrt_llm8executor6TensorEEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine10getAddressEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine10getAddressEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine10getAddressEv"></span><span id="tensorrt_llm::runtime::RawEngine::getAddressC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a9802245035ff5d5b1e15fddf08d47b7a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getAddress</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine10getAddressEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine7getSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine7getSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine7getSizeEv"></span><span id="tensorrt_llm::runtime::RawEngine::getSizeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a111cdd7dd515f0692199ae815f0aa186"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime9RawEngine13getHostMemoryEv">
<span id="_CPPv3NK12tensorrt_llm7runtime9RawEngine13getHostMemoryEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime9RawEngine13getHostMemoryEv"></span><span id="tensorrt_llm::runtime::RawEngine::getHostMemoryC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a82d55c1942b25bbad111fff71336066e"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">IHostMemory</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getHostMemory</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine13getHostMemoryEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-members">Public Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineAddrE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine11mEngineAddrE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine11mEngineAddrE"></span><span id="tensorrt_llm::runtime::RawEngine::mEngineAddr__voidCP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a95a24a9a7ada011d7f6260f5948098a1"></span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">mEngineAddr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineAddrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineSizeE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine11mEngineSizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine11mEngineSizeE"></span><span id="tensorrt_llm::runtime::RawEngine::mEngineSize__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a3c29c2fc9109c3155a75346a3ba4b859"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEngineSize</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineSizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine5mTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine5mTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine5mTypeE"></span><span id="tensorrt_llm::runtime::RawEngine::mType__Type"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1ae7f9d965ecc951e7eab03a556d5079ac"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4TypeE" title="tensorrt_llm::runtime::RawEngine::Type"><span class="n"><span class="pre">Type</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine5mTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine11mEnginePathE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine11mEnginePathE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine11mEnginePathE"></span><span id="tensorrt_llm::runtime::RawEngine::mEnginePath__std::optional:std::filesystem::path:"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1ae0a8b43561f46764487b26a60956a44d"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">filesystem</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">path</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mEnginePath</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEnginePathE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp">
<span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a4b1eb35ae9e6aad50395b67a1d91c04f"></span><span class="sig-name descname"><span class="pre">struct</span> <span class="pre">tensorrt_llm::runtime::RawEngine</span></span></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine13mEngineBufferE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine13mEngineBufferE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine13mEngineBufferE"></span><span id="tensorrt_llm::runtime::RawEngine::mEngineBuffer__nvinfer1::IHostMemoryCP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a991cf95f70001c61a7062bf62a1684ae"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">IHostMemory</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">mEngineBuffer</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="p"><span class="pre">{</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine13mEngineBufferE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime9RawEngine18mManagedWeightsMapE">
<span id="_CPPv3N12tensorrt_llm7runtime9RawEngine18mManagedWeightsMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime9RawEngine18mManagedWeightsMapE"></span><span id="tensorrt_llm::runtime::RawEngine::mManagedWeightsMap__std::optional:std::map:ss.tensorrt_llm::executor::Tensor::"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1RawEngine_1a48f7d27244eacf245b02d90ded15f83e"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">map</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executorE" title="tensorrt_llm::executor"><span class="n"><span class="pre">executor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm8executor6TensorE" title="tensorrt_llm::executor::Tensor"><span class="n"><span class="pre">Tensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mManagedWeightsMap</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine18mManagedWeightsMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="itensor-h">
<h2>iTensor.h<a class="headerlink" href="#itensor-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv48nvinfer1">
<span id="_CPPv38nvinfer1"></span><span id="_CPPv28nvinfer1"></span><span id="nvinfer1"></span><span class="target" id="namespacenvinfer1"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">nvinfer1</span></span></span><a class="headerlink" href="#_CPPv48nvinfer1" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN7ITensor5ShapeE">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERKN7ITensor5ShapeE"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERKN7ITensor5ShapeE"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.ITensor::ShapeCR"></span><span class="target" id="iTensor_8h_1a0c96496836cd2bdc559321f5e7c31294"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN7ITensor5ShapeE" title="Link to this definition">#</a><br /></dt>
<dd><p>Utility function to print a shape. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7ITensor">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERK7ITensor"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERK7ITensor"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.ITensorCR"></span><span class="target" id="iTensor_8h_1a9169197d73d3cf134f280862c275af26"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7ITensor" title="Link to this definition">#</a><br /></dt>
<dd><p>Utility function to print a tensor with its shape. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7ITensor14SharedConstPtrE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7ITensor14SharedConstPtrE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7ITensor14SharedConstPtrE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iTensor_8h_1a52b485dcf9191f3234857aea11a34e09"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensorPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7ITensor14SharedConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T const typed pointer to the underlying data of the tensor pointed to by the tensorPtr, or nullptr if the tensorPtr is null. </p>
<p>This overload has to be declared to avoid ambiguity when an implicit conversion to <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref">IBuffer</span></a> is involved. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>tensorPtr</strong> – A possibly null shared ptr. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to T const, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7ITensor9SharedPtrE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7ITensor9SharedPtrE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7ITensor9SharedPtrE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iTensor_8h_1a7167b962238a8d59bcd7187f3c35595b"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7ITensor9SharedPtrE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensorPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7ITensor9SharedPtrE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T typed pointer to the underlying data of the buffer pointed to by the tensorPtr, or nullptr if the tensorPtr is null. </p>
<p>This overload has to be declared to avoid ambiguity when an implicit conversion to <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref">IBuffer</span></a> is involved. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>tensorPtr</strong> – A possibly null shared ptr. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7ITensor9SharedPtrEEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7ITensor9SharedPtrEEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7ITensor9SharedPtrEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iTensor_8h_1ad9a080040e1bda831a207038efb156d7"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7ITensor9SharedPtrEEE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">optionalTensorPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7ITensor9SharedPtrEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T typed pointer to the underlying data of the tensor pointed to by the tensor pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value. </p>
<p>This overload has to be declared to avoid ambiguity when an implicit conversion to <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref">IBuffer</span></a> is involved. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>optionalBufferPtr</strong> – A possibly empty optional. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7ITensor14SharedConstPtrEEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7ITensor14SharedConstPtrEEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7ITensor14SharedConstPtrEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iTensor_8h_1afc77303772dfea61ffd36f5fa06e73e6"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7ITensor14SharedConstPtrEEE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="tensorrt_llm::runtime::ITensor::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">optionalTensorPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7ITensor14SharedConstPtrEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T const typed pointer to the underlying data of the tensor pointed to by the tensor pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value. </p>
<p>This overload has to be declared to avoid ambiguity when an implicit conversion to <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref">IBuffer</span></a> is involved. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>optionalBufferPtr</strong> – A possibly empty optional. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to const T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensorE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensorE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensorE"></span><span id="tensorrt_llm::runtime::ITensor"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">ITensor</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9UniquePtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9UniquePtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a0ebcd2ee37abad2960daf1346816f348"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UniquePtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9SharedPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9SharedPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a31e67140fadaeb6eaaf096ff37d640ad"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SharedPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ad199f95dd25dfbef298cc7d1f3655d66"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UniqueConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor14SharedConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor14SharedConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a591e6b0aa01a0043967262ffdcc3974d"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SharedConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5ShapeE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5ShapeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ab2e78d36e5cc72b4443e832a46147fcc"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Shape</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">Dims</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9DimType64E"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9DimType64E"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a0e1b3d629571428c37c511b70a05b334"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DimType64</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_reference_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">decltype</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">d</span></span><span class="p"><span class="pre">[</span></span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9TensorMapE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9TensorMapE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9TensorMapE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ada89b277ed3347bbb527e3092a8eab04"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TensorMap</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE" title="tensorrt_llm::runtime::StringPtrMap"><span class="n"><span class="pre">StringPtrMap</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9TensorMapE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensorD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensorD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensorD0Ev"></span><span id="tensorrt_llm::runtime::ITensor::~ITensor"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a857da95b81330009b368308ed0c2d3d9"></span><span class="sig-name descname"><span class="n"><span class="pre">~ITensor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensorD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7ITensor8getShapeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7ITensor8getShapeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7ITensor8getShapeEv"></span><span id="tensorrt_llm::runtime::ITensor::getShapeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a8efdcf81157ec42fe414411891370355"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="n"><span class="pre">getShape</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor8getShapeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the tensor dimensions. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I_10SizeType32ENK12tensorrt_llm7runtime7ITensor12getDimensionE9DimType64v">
<span id="_CPPv3I_10SizeType32ENK12tensorrt_llm7runtime7ITensor12getDimensionEv"></span><span id="_CPPv2I_10SizeType32ENK12tensorrt_llm7runtime7ITensor12getDimensionEv"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">n</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a9d9640edefab8053cd81ed3e96bedd32"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDimension</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I_10SizeType32ENK12tensorrt_llm7runtime7ITensor12getDimensionE9DimType64v" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the tensor n-th dimension. If n is negative, returns the (nbDims - n)th dimension. TODO: replace with constexpr parameter when moving to C++20. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7reshapeERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7reshapeERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7reshapeERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::reshape__ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a772377264b089406fb1cf743ac3163f3"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">reshape</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7reshapeERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>Sets the tensor dimensions. The new size of the tensor will be <code class="docutils literal notranslate"><span class="pre">volume(dims)</span></code></p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor6resizeENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor6resizeENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor6resizeENSt6size_tE"></span><span id="tensorrt_llm::runtime::ITensor::resize__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a8ed7ae223a11d8d516216a4d28286f62"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">resize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">newSize</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">override</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor6resizeENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Resizes the buffer. This is a no-op if the new size is smaller than or equal to the current capacity. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor"></span><span id="tensorrt_llm::runtime::ITensor::ITensor__ITensorCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ade4b1d2a214d378f73255357e32bee1f"></span><span class="sig-name descname"><span class="n"><span class="pre">ITensor</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor" title="tensorrt_llm::runtime::ITensor::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor" title="Link to this definition">#</a><br /></dt>
<dd><p>Not allowed to copy. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensoraSERK7ITensor">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensoraSERK7ITensor"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensoraSERK7ITensor"></span><span id="tensorrt_llm::runtime::ITensor::assign-operator__ITensorCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1afa9496881fe6d311048891210d206a83"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensoraSERK7ITensor" title="Link to this definition">#</a><br /></dt>
<dd><p>Not allowed to copy. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7squeezeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7squeezeE10SizeType32"></span><span id="tensorrt_llm::runtime::ITensor::squeeze__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a19b6e0dda1badc00eee1c8d558a627bd"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">squeeze</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dim</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Removes the given <em>unit</em> dimensions from this tensor. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeE10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9unsqueezeE10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9unsqueezeE10SizeType32"></span><span id="tensorrt_llm::runtime::ITensor::unsqueeze__SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a93cd46c1307565a785482a58a454f5a1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">unsqueeze</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dim</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeE10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Adds a <em>unit</em> dimension at the specified position. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERK5Shape">
<span id="_CPPv3NK12tensorrt_llm7runtime7ITensor11shapeEqualsERK5Shape"></span><span id="_CPPv2NK12tensorrt_llm7runtime7ITensor11shapeEqualsERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::shapeEquals__ShapeCRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ab84847a01479731500363a637003e03a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shapeEquals</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">other</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERKNSt16initializer_listI10SizeType32EE">
<span id="_CPPv3NK12tensorrt_llm7runtime7ITensor11shapeEqualsERKNSt16initializer_listI10SizeType32EE"></span><span id="_CPPv2NK12tensorrt_llm7runtime7ITensor11shapeEqualsERKNSt16initializer_listI10SizeType32EE"></span><span id="tensorrt_llm::runtime::ITensor::shapeEquals__std::initializer_list:SizeType32:CRC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1acc94dcf55ed9f128a97581822d735239"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shapeEquals</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">other</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERKNSt16initializer_listI10SizeType32EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEbPK1T10SizeType32">
<span id="_CPPv3I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEPK1T10SizeType32"></span><span id="_CPPv2I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEPK1T10SizeType32"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a90290f101c7fccb5719d975820f861a0"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shapeEquals</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEbPK1T10SizeType32" title="tensorrt_llm::runtime::ITensor::shapeEquals::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">count</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEbPK1T10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor6volumeERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor6volumeERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor6volumeERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::volume__ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a2afffa26551b43982bf51fa4920d4ade"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">volume</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor6volumeERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the volume of the dimensions. Returns -1 if <code class="docutils literal notranslate"><span class="pre">d.nbDims</span> <span class="pre">&lt;</span> <span class="pre">0</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor17volumeNonNegativeERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor17volumeNonNegativeERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor17volumeNonNegativeERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::volumeNonNegative__ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1aac34c9309ce068d8216a54343348e21d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">volumeNonNegative</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor17volumeNonNegativeERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the volume of the dimensions. Throws if <code class="docutils literal notranslate"><span class="pre">d.nbDims</span> <span class="pre">&lt;</span> <span class="pre">0</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7stridesERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7stridesERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7stridesERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::strides__ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a9a9b3844320d4e08705c892320ef526c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">strides</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7stridesERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the strides of each dimemsion in a Shape. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeERK5Shape10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7squeezeERK5Shape10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7squeezeERK5Shape10SizeType32"></span><span id="tensorrt_llm::runtime::ITensor::squeeze__ShapeCR.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1adb620fc3a8eeac8e8502f8e864e3d822"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">squeeze</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dim</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeERK5Shape10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Removes the given <em>unit</em> dimension from <code class="docutils literal notranslate"><span class="pre">shape</span></code>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>shape</strong> – The shape to squeeze. </p></li>
<li><p><strong>dim</strong> – The dimension that should be removed (“squeezed”). </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A new shape without the unit dimension. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeERK5Shape10SizeType32">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9unsqueezeERK5Shape10SizeType32"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9unsqueezeERK5Shape10SizeType32"></span><span id="tensorrt_llm::runtime::ITensor::unsqueeze__ShapeCR.SizeType32"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1af34ca957b146fa051184160c1db78af0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">unsqueeze</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dim</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeERK5Shape10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>Add a <em>unit</em> dimension to <code class="docutils literal notranslate"><span class="pre">shape</span></code> at the specified position. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>shape</strong> – The shape to unsqueeze. </p></li>
<li><p><strong>dim</strong> – The dimension where unit dimension should be added. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A new shape with the added unit dimension. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tENSt6size_tE"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.std::s.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1aa22bf50eff27dbfc64eed003851f4778"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Creates a sliced view on the underlying <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. The view will have the same data type as <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – The tensor to view. </p></li>
<li><p><strong>offset</strong> – The offset of the view w.r.t. dimension 0 of the tensor. </p></li>
<li><p><strong>size</strong> – The size of the view w.r.t. dimension 0 of the tensor. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">buffer</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrNSt6size_tENSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrNSt6size_tENSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a201a60f7da459ec35f5d750e8d7053c4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tE"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a3bc0148d89f892df6beff0f63ea33e6a"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrNSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a01d828a32b3f280c1ee64ce5211da20b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape9DimType64">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape9DimType64"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape9DimType64"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.ShapeCR.DimType64"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a60cd2e9b1d7c91a1201510f8deec9248"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape9DimType64" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>offsetDims</strong> – The offset in multiple dimensions.</p></li>
<li><p><strong>tensor</strong> – The tensor to view. </p></li>
<li><p><strong>offsetDims</strong> – The offset dimensions of the view. </p></li>
<li><p><strong>size</strong> – The size of the view w.r.t. the last dimension in offsetDims. </p></li>
<li><p><strong>offsetDims</strong> – specifies all dimensions. </p></li>
</ul>
</dd>
<dt class="field-even">Throws<span class="colon">:</span></dt>
<dd class="field-even"><p><span><span class="cpp-expr sig sig-inline cpp"><span class="n">Whenever</span></span></span> – offset overflows or the last dimension offset+size overflows. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A view of shape [size, the rest dimensions] or [size] when</p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE9DimType64">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE9DimType64"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE9DimType64"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.std::initializer_list:DimType64:CR.DimType64"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a1f486293132bbe0ac17a430db3d8c87c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE9DimType64" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5ShapeNSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRK5ShapeNSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRK5ShapeNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5ShapeNSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a2d0ab0192c27695bedeb7134cad400ac"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5ShapeNSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5ShapeNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ad35b215bf9822c1e21c77647c78ba4ec"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ade7e339e615cd675e4201903b3d28139"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>return the rest slices at the last dimension when <code class="docutils literal notranslate"><span class="pre">size</span></code> omitted. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE"></span><span id="tensorrt_llm::runtime::ITensor::slice__SharedPtr.std::initializer_list:DimType64:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a612d0d2c27d219f5344a5acb244faca3"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5Shape">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRK5Shape"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRK5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1aaad4e79382fcd3fc2f55c0f4d088dc74"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRKNSt16initializer_listI9DimType64EE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceERR9TConstPtrRKNSt16initializer_listI9DimType64EE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1aa61976020990ef44546e22921156dd88"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="tensorrt_llm::runtime::ITensor::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::at__SharedPtr.ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a02711c850a3a20114316a76ac00593a1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">at</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>offsetDims</strong> – specifies all dimensions. </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>Just the block at the point, with shape of [the rest dimensions] or [1] when</p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRKNSt16initializer_listI9DimType64EE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRKNSt16initializer_listI9DimType64EE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRKNSt16initializer_listI9DimType64EE"></span><span id="tensorrt_llm::runtime::ITensor::at__SharedPtr.std::initializer_list:DimType64:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a920d45c26d8b49480daf695779c67183"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">at</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRKNSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atE14UniqueConstPtrRR9TConstPtrRK5Shape">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atERR9TConstPtrRK5Shape"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atERR9TConstPtrRK5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::at::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ab0e745de500a91974a20dd31783f3efb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">at</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::at::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atE14UniqueConstPtrRR9TConstPtrRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atEN7ITensor14UniqueConstPtrERR9TConstPtrRKNSt16initializer_listI9DimType64EE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atERR9TConstPtrRKNSt16initializer_listI9DimType64EE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atERR9TConstPtrRKNSt16initializer_listI9DimType64EE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atEN7ITensor14UniqueConstPtrERR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="tensorrt_llm::runtime::ITensor::at::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a60ebb8d28d2e24cf31f941645b237552"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE" title="tensorrt_llm::runtime::ITensor"><span class="n"><span class="pre">ITensor</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">at</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atEN7ITensor14UniqueConstPtrERR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="tensorrt_llm::runtime::ITensor::at::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">offsetDims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atEN7ITensor14UniqueConstPtrERR9TConstPtrRKNSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor4viewEN7IBuffer9SharedPtrERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor4viewEN7IBuffer9SharedPtrERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor4viewEN7IBuffer9SharedPtrERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::view__IBuffer::SharedPtr.ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1aeb7a31c92f9ebc17e4648ba4fff7e16e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">buffer</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4viewEN7IBuffer9SharedPtrERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a view on the underlying <code class="docutils literal notranslate"><span class="pre">buffer</span></code> (or tensor) with the given shape. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – The tensor to view. </p></li>
<li><p><strong>shape</strong> – The shape of the view. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewE14UniqueConstPtrRR9TConstPtrRK5Shape">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewERR9TConstPtrRK5Shape"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewERR9TConstPtrRK5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::view::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a481831d7c0d45fe462e91b07beef2617"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE" title="tensorrt_llm::runtime::ITensor::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewE14UniqueConstPtrRR9TConstPtrRK5Shape" title="tensorrt_llm::runtime::ITensor::view::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewE14UniqueConstPtrRR9TConstPtrRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor4viewE9SharedPtr">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor4viewE9SharedPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor4viewE9SharedPtr"></span><span id="tensorrt_llm::runtime::ITensor::view__SharedPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a0d748a6fdb8219af29ba918878bf44b2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4viewE9SharedPtr" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a view on the underlying <code class="docutils literal notranslate"><span class="pre">tensor</span></code> which can be independently reshaped. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>tensor</strong> – The tensor to view. </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor8flattenNE9SharedPtrNSt7int64_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor8flattenNE9SharedPtrNSt7int64_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor8flattenNE9SharedPtrNSt7int64_tE"></span><span id="tensorrt_llm::runtime::ITensor::flattenN__SharedPtr.std::int64_t"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a7078d3b5b40ba1257c3be030e4766c9e"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">flattenN</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE" title="tensorrt_llm::runtime::ITensor::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">sliceN</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="o"><span class="pre">-</span></span><span class="m"><span class="pre">1</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8flattenNE9SharedPtrNSt7int64_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a flattened view on the underlying <code class="docutils literal notranslate"><span class="pre">tensor</span></code> which can be independently reshaped. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – The tensor to flatten. </p></li>
<li><p><strong>sliceN</strong> – Slice the first N elements after flattening. -1 means take the whole flattened tensor. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A flatten view on the <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5ShapeNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5ShapeNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5ShapeNSt6size_tE"></span><span id="tensorrt_llm::runtime::ITensor::wrap__voidP.nvinfer1::DataType.ShapeCR.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1abb208b99fc6a404a13215fe2d43637a4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">capacity</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5ShapeNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Wraps the given <code class="docutils literal notranslate"><span class="pre">data</span></code> in an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code>. The <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code> will not own the underlying <code class="docutils literal notranslate"><span class="pre">data</span></code> and cannot be reshaped beyond <code class="docutils literal notranslate"><span class="pre">capacity</span></code>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>data</strong> – The data to wrap. </p></li>
<li><p><strong>type</strong> – The data type of the <code class="docutils literal notranslate"><span class="pre">data</span></code>. </p></li>
<li><p><strong>shape</strong> – The shape of the tensor. </p></li>
<li><p><strong>capacity</strong> – The capacity of the buffer. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>An <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref"><span class="pre">ITensor</span></span></a></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::wrap__voidP.nvinfer1::DataType.ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a5fdde0b2cc5bb0ee07c042964bb48c85"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5ShapeNSt6size_tE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7ITensor4wrapEP1TRK5ShapeNSt6size_tE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7ITensor4wrapEP1TRK5ShapeNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a6cb1dc1fe13779d7d0bed2062df2c0e0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5ShapeNSt6size_tE" title="tensorrt_llm::runtime::ITensor::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">capacity</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5ShapeNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5Shape">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7ITensor4wrapEP1TRK5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7ITensor4wrapEP1TRK5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a496e23f8dcc29f792a7e0f70fca11973"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5Shape" title="tensorrt_llm::runtime::ITensor::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrRNSt6vectorI1TEERK5Shape">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7ITensor4wrapERNSt6vectorI1TEERK5Shape"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7ITensor4wrapERNSt6vectorI1TEERK5Shape"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ab27a7a0ae81205699d2b2bc618367857"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE" title="tensorrt_llm::runtime::ITensor::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrRNSt6vectorI1TEERK5Shape" title="tensorrt_llm::runtime::ITensor::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">v</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">shape</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrRNSt6vectorI1TEERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor9makeShapeERKNSt16initializer_listI9DimType64EE">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor9makeShapeERKNSt16initializer_listI9DimType64EE"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor9makeShapeERKNSt16initializer_listI9DimType64EE"></span><span id="tensorrt_llm::runtime::ITensor::makeShape__std::initializer_list:DimType64:CR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a6ae0b710b39f4f5d821681188c8afce4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">makeShape</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">initializer_list</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9makeShapeERKNSt16initializer_listI9DimType64EE" title="Link to this definition">#</a><br /></dt>
<dd><p>A convenience function to create a tensor shape with the given dimensions. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor8toStringERK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor8toStringERK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor8toStringERK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::toString__ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1af9ba06b8cbf144ea573e6767af85bd43"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">string</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toString</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">dims</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8toStringERK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>A convenience function for converting a tensor shape to a <code class="docutils literal notranslate"><span class="pre">string</span></code>. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapeRK5Shape">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapeRK5Shape"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapeRK5Shape"></span><span id="tensorrt_llm::runtime::ITensor::shapeEquals__ShapeCR.ShapeCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1ae39159cc236f70454983d0193570e1c4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shapeEquals</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">lhs</span></span></em>, <em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">rhs</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapeRK5Shape" title="Link to this definition">#</a><br /></dt>
<dd><p>A convenience function to compare shapes. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsEbRK5ShapePK1T10SizeType32">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapePK1T10SizeType32"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapePK1T10SizeType32"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1afbf1a7c0ee57cf176f6dfbb04766b9d9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">shapeEquals</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE" title="tensorrt_llm::runtime::ITensor::Shape"><span class="n"><span class="pre">Shape</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">lhs</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsEbRK5ShapePK1T10SizeType32" title="tensorrt_llm::runtime::ITensor::shapeEquals::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">dims</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E" title="tensorrt_llm::runtime::SizeType32"><span class="n"><span class="pre">SizeType32</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">count</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsEbRK5ShapePK1T10SizeType32" title="Link to this definition">#</a><br /></dt>
<dd><p>A convenience function to compare shapes. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-protected-functions">Protected Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorEv">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor7ITensorEv"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor7ITensorEv"></span><span id="tensorrt_llm::runtime::ITensor::ITensor"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a60742a4752e9d9d5a82fff9e24518953"></span><span class="sig-name descname"><span class="n"><span class="pre">ITensor</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-protected-static-functions">Protected Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7ITensor8castSizeE6size_t">
<span id="_CPPv3N12tensorrt_llm7runtime7ITensor8castSizeE6size_t"></span><span id="_CPPv2N12tensorrt_llm7runtime7ITensor8castSizeE6size_t"></span><span id="tensorrt_llm::runtime::ITensor::castSize__s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1ITensor_1a0d4cb08fc550e2b0e97399a76e3d64ec"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E" title="tensorrt_llm::runtime::ITensor::DimType64"><span class="n"><span class="pre">DimType64</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">castSize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">newSize</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8castSizeE6size_t" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-friends">Friends</p>
<dl>
<dt class="sig sig-object cpp">
<em class="property"><span class="pre">friend</span> <span class="pre">class</span></em> <span class="pre">ITensorBindings</span></dt>
</dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
<section id="ibuffer-h">
<h2>iBuffer.h<a class="headerlink" href="#ibuffer-h" title="Link to this heading">#</a></h2>
<dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">tensorrt_llm</span></span></span><br /></dt>
<dd><dl class="cpp type">
<dt class="sig sig-object cpp">
<span class="target" id="namespacetensorrt__llm_1_1runtime"></span><span class="k"><span class="pre">namespace</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">runtime</span></span></span><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-typedefs">Typedefs</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime18PointerElementTypeE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime18PointerElementTypeE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a6b78d5482ec51a801a9fe54db6eaa0f9"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">PointerElementType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_reference_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">element_type</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-enums">Enums</p>
<dl class="cpp enum-class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryTypeE"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869"></span><span class="k"><span class="pre">enum</span></span><span class="w"> </span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryType</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="Link to this definition">#</a><br /></dt>
<dd><p><em>Values:</em></p>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryType4kGPUE">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryType4kGPUE"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryType4kGPUE"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869a37ae524b76587efa776affdc5cdf2ac1"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kGPU</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kGPUE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryType4kCPUE">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryType4kCPUE"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryType4kCPUE"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869a2ce310327f474afc9f6774faa2f57903"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kCPU</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kCPUE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryType7kPINNEDE">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryType7kPINNEDE"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryType7kPINNEDE"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869ae3278bcaa387e6baeef9b80c1e61c35a"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kPINNED</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType7kPINNEDE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryType4kUVME">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryType4kUVME"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryType4kUVME"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869a7d7de0bd70e1276255098b25010bdeb6"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUVM</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kUVME" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp enumerator">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE">
<span id="_CPPv3N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE"></span><span id="_CPPv2N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE"></span><span class="target" id="iBuffer_8h_1a57051a59520fc335a72e6fd1d08fa869ac61fd7eec16ee67dfabffa6b6c7dd8aa"></span><span class="k"><span class="pre">enumerator</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kPINNEDPOOL</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-functions">Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERKNSt10shared_ptrI1TEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16constPointerCastERKNSt10shared_ptrI1TEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16constPointerCastERKNSt10shared_ptrI1TEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1ae4deb6e27fee100e03742f3559ec25e7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_const_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERKNSt10shared_ptrI1TEE" title="tensorrt_llm::runtime::constPointerCast::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">constPointerCast</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERKNSt10shared_ptrI1TEE" title="tensorrt_llm::runtime::constPointerCast::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">ptr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERKNSt10shared_ptrI1TEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE">
<span id="_CPPv3I00EN12tensorrt_llm7runtime16constPointerCastERRNSt10unique_ptrI1T1DEE"></span><span id="_CPPv2I00EN12tensorrt_llm7runtime16constPointerCastERRNSt10unique_ptrI1T1DEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">D</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a1ea3ede13d88a2910f88551b802932b8"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_const_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE" title="tensorrt_llm::runtime::constPointerCast::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">constPointerCast</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE" title="tensorrt_llm::runtime::constPointerCast::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE" title="tensorrt_llm::runtime::constPointerCast::D"><span class="n"><span class="pre">D</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">ptr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEPK1TRK7IBuffer">
<span id="_CPPv3I0EN12tensorrt_llm7runtime10bufferCastERK7IBuffer"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime10bufferCastERK7IBuffer"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a582318a59a26e5e20f230a7d7c6a0d9f"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEPK1TRK7IBuffer" title="tensorrt_llm::runtime::bufferCast::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCast</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEPK1TRK7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Gets a typed pointer to the constant underlying data of the buffer. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>buffer</strong> – The buffer to get a pointer to. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to constant <code class="docutils literal notranslate"><span class="pre">T</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEP1TR7IBuffer">
<span id="_CPPv3I0EN12tensorrt_llm7runtime10bufferCastER7IBuffer"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime10bufferCastER7IBuffer"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a650f806d6eabc04b0cad8a5cdc888f4d"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEP1TR7IBuffer" title="tensorrt_llm::runtime::bufferCast::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCast</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEP1TR7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Gets a typed pointer to the underlying data of the buffer. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>buffer</strong> – The buffer to get a pointer to. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to <code class="docutils literal notranslate"><span class="pre">T</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7IBuffer9SharedPtrE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7IBuffer9SharedPtrE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7IBuffer9SharedPtrE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a3d2548426cec96db1d96d3dc2485431c"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7IBuffer9SharedPtrE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T typed pointer to the underlying data of the buffer pointed to by the bufferPtr, or nullptr if the bufferPtr is null. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>bufferPtr</strong> – A possibly null shared ptr. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7IBuffer14SharedConstPtrE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7IBuffer14SharedConstPtrE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKN7IBuffer14SharedConstPtrE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1ae558ab980f04f24da40bde948f7e27b2"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7IBuffer14SharedConstPtrE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE" title="tensorrt_llm::runtime::IBuffer::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">bufferPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7IBuffer14SharedConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T const typed pointer to the underlying data of the buffer pointed to by the bufferPtr, or nullptr if the bufferPtr is null. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>bufferPtr</strong> – A possibly null shared ptr. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to const T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7IBuffer9SharedPtrEEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7IBuffer9SharedPtrEEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7IBuffer9SharedPtrEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1a12d75e707c282d3abdd69933038489d6"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7IBuffer9SharedPtrEEE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">optionalBufferPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7IBuffer9SharedPtrEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T typed pointer to the underlying data of the buffer pointed to by the buffer pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>optionalBufferPtr</strong> – A possibly empty optional. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7IBuffer14SharedConstPtrEEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7IBuffer14SharedConstPtrEEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime16bufferCastOrNullERKNSt8optionalIN7IBuffer14SharedConstPtrEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="iBuffer_8h_1aa1001e409507709767368842e3c6b0c7"></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7IBuffer14SharedConstPtrEEE" title="tensorrt_llm::runtime::bufferCastOrNull::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">bufferCastOrNull</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">optional</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE" title="tensorrt_llm::runtime::IBuffer::SharedConstPtr"><span class="n"><span class="pre">SharedConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">optionalBufferPtr</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7IBuffer14SharedConstPtrEEE" title="Link to this definition">#</a><br /></dt>
<dd><p>Retrieves a T const typed pointer to the underlying data of the buffer pointed to by the buffer pointer contained in the optionalBufferPtr, or nullptr if the optional doesn’t have a value. </p>
<dl class="field-list simple">
<dt class="field-odd">Template Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>T</strong> – The type of the underlying data. </p>
</dd>
<dt class="field-even">Parameters<span class="colon">:</span></dt>
<dd class="field-even"><p><strong>optionalBufferPtr</strong> – A possibly empty optional. </p>
</dd>
<dt class="field-odd">Returns<span class="colon">:</span></dt>
<dd class="field-odd"><p>A pointer to const T, possibly nullptr. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7IBuffer">
<span id="_CPPv3N12tensorrt_llm7runtimelsERNSt7ostreamERK7IBuffer"></span><span id="_CPPv2N12tensorrt_llm7runtimelsERNSt7ostreamERK7IBuffer"></span><span id="tensorrt_llm::runtime::lshift-operator__osR.IBufferCR"></span><span class="target" id="iBuffer_8h_1a56e1be5ed31af23ee77c9001f938c1d7"></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">&lt;&lt;</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ostream</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">output</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Utility function to print a buffer. </p>
</dd></dl>
</div>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataTypeE"></span><span id="tensorrt_llm::runtime::BufferDataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferDataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;iBuffer.h&gt;</em></div>
<p>A wrapper around <code class="docutils literal notranslate"><span class="pre">nvinfer1::DataType</span></code> that provides a support for pointer types. </p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataType14BufferDataTypeEN8nvinfer18DataTypeEbb">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataType14BufferDataTypeEN8nvinfer18DataTypeEbb"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataType14BufferDataTypeEN8nvinfer18DataTypeEbb"></span><span id="tensorrt_llm::runtime::BufferDataType::BufferDataType__nvinfer1::DataType.b.bCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a3cdb7e5d96b677f670d211cd3f0d2029"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferDataType</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">_unsigned</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
<dd><em class="sig-param"><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">pointer</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType14BufferDataTypeEN8nvinfer18DataTypeEbb" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataTypecvN8nvinfer18DataTypeEEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataTypecvN8nvinfer18DataTypeEEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataTypecvN8nvinfer18DataTypeEEv"></span><span id="tensorrt_llm::runtime::BufferDataType::castto-nvinfer1::DataType-operatorCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1ae0e234f2e49d57ea1ec39bf76ef5f843"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataTypecvN8nvinfer18DataTypeEEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataType11getDataTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataType11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataType11getDataTypeEv"></span><span id="tensorrt_llm::runtime::BufferDataType::getDataTypeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a375ef9858082b42a89a8fbddeb758836"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataType9isPointerEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataType9isPointerEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataType9isPointerEv"></span><span id="tensorrt_llm::runtime::BufferDataType::isPointerCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a4f456474dbc06fcfb4c5fdd4f30c681b"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isPointer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType9isPointerEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataType10isUnsignedEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataType10isUnsignedEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataType10isUnsignedEv"></span><span id="tensorrt_llm::runtime::BufferDataType::isUnsignedCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a84da512a92aa9b9ba7498d8eface36d1"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">isUnsigned</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType10isUnsignedEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataType7getSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataType7getSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataType7getSizeEv"></span><span id="tensorrt_llm::runtime::BufferDataType::getSizeCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a0e1c8d7eb345611735b5c8998efc1b29"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType7getSizeEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime14BufferDataType13getSizeInBitsEv">
<span id="_CPPv3NK12tensorrt_llm7runtime14BufferDataType13getSizeInBitsEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime14BufferDataType13getSizeInBitsEv"></span><span id="tensorrt_llm::runtime::BufferDataType::getSizeInBitsCCE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a42aaae0de1ad95e54048fa773c1cf1e8"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSizeInBits</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="k"><span class="pre">noexcept</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType13getSizeInBitsEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE"></span><span id="tensorrt_llm::runtime::BufferDataType::kTrtPointerType__auto"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1ac186826cc80422eae28ba74309100cd9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kTrtPointerType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT64</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-members">Private Members</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataType9mDataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataType9mDataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataType9mDataTypeE"></span><span id="tensorrt_llm::runtime::BufferDataType::mDataType__nvinfer1::DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a57916add2426171d7066d14df4e796d8"></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mDataType</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType9mDataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataType9mUnsignedE">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataType9mUnsignedE"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataType9mUnsignedE"></span><span id="tensorrt_llm::runtime::BufferDataType::mUnsigned__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a675fe3124bb2de4dd047ce69a1a5a88b"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mUnsigned</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType9mUnsignedE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14BufferDataType8mPointerE">
<span id="_CPPv3N12tensorrt_llm7runtime14BufferDataType8mPointerE"></span><span id="_CPPv2N12tensorrt_llm7runtime14BufferDataType8mPointerE"></span><span id="tensorrt_llm::runtime::BufferDataType::mPointer__b"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferDataType_1a494f3b4c8e08ef43209cac0e2f114e0c"></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">mPointer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType8mPointerE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime11BufferRangeE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime11BufferRangeE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferRange"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferRange</span></span></span><span class="w"> </span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="k"><span class="pre">public</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ArrayView</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="tensorrt_llm::runtime::BufferRange::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11BufferRange4BaseE">
<span id="_CPPv3N12tensorrt_llm7runtime11BufferRange4BaseE"></span><span id="_CPPv2N12tensorrt_llm7runtime11BufferRange4BaseE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferRange_1a58ffed28eecc57b7ad0178ce58cdaa61"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">Base</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="executor.html#_CPPv412tensorrt_llm" title="tensorrt_llm"><span class="n"><span class="pre">tensorrt_llm</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">common</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">ArrayView</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="tensorrt_llm::runtime::BufferRange::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11BufferRange4BaseE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11BufferRange11BufferRangeEP1T9size_type">
<span id="_CPPv3N12tensorrt_llm7runtime11BufferRange11BufferRangeEP1T9size_type"></span><span id="_CPPv2N12tensorrt_llm7runtime11BufferRange11BufferRangeEP1T9size_type"></span><span id="tensorrt_llm::runtime::BufferRange::BufferRange__TP.size_type"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferRange_1a7ea39c3ef4a0a2bf24228697045fc145"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferRange</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="tensorrt_llm::runtime::BufferRange::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">size_type</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11BufferRange11BufferRangeEP1T9size_type" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tIXntNSt10is_const_vI1UEEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer">
<span id="_CPPv3I0_NSt11enable_if_tIXntNSt10is_const_vI1UEEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer"></span><span id="_CPPv2I0_NSt11enable_if_tIX!std::is_const_v<U>EbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">U</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="tensorrt_llm::runtime::BufferRange::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="o"><span class="pre">!</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tIXntNSt10is_const_vI1UEEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer" title="tensorrt_llm::runtime::BufferRange::BufferRange::U"><span class="n"><span class="pre">U</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferRange_1aa31b97783316f3d1e92432c0753c9e99"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferRange</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tIXntNSt10is_const_vI1UEEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">U</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE" title="tensorrt_llm::runtime::BufferRange::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer" title="tensorrt_llm::runtime::BufferRange::BufferRange::U"><span class="n"><span class="pre">U</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1BufferRange_1a5bb8dc187700342fb7254683285de05a"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">explicit</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">BufferRange</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I_N8nvinfer18DataTypeE_b_bEN12tensorrt_llm7runtime14DataTypeTraitsE">
<span id="_CPPv3I_N8nvinfer18DataTypeE_b_bEN12tensorrt_llm7runtime14DataTypeTraitsE"></span><span id="_CPPv2I_N8nvinfer18DataTypeE_b_bEN12tensorrt_llm7runtime14DataTypeTraitsE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kDataType</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kIsUnsigned</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kIsPointer</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><a class="headerlink" href="#_CPPv4I_N8nvinfer18DataTypeE_b_bEN12tensorrt_llm7runtime14DataTypeTraitsE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;iBuffer.h&gt;</em></div>
<p>For converting a TensorRT data type to a C++ data type. </p>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE">
<span id="_CPPv3I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE"></span><span id="_CPPv2I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXtrueEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kDataType</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kUnsigned</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01kDataType_00_01kUnsigned_00_01true_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;kDataType, kUnsigned, true&gt;::kDataType"><span class="n"><span class="pre">kDataType</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;kDataType, kUnsigned, true&gt;::kUnsigned"><span class="n"><span class="pre">kUnsigned</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXtrueEE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01kDataType_00_01kUnsigned_00_01true_01_4_1a4e4f2844614eff1320acd710bc0cfe9c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="n"><span class="pre">DataTypeTraits</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;kDataType, kUnsigned, true&gt;::kDataType"><span class="n"><span class="pre">kDataType</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;kDataType, kUnsigned, true&gt;::kUnsigned"><span class="n"><span class="pre">kUnsigned</span></span></a><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">type</span></span><span class="p"><span class="pre">*</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXtrueEE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:kDataType.kUnsigned.true:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01kDataType_00_01kUnsigned_00_01true_01_4_1a3a25a81fd2f6753350919c8a0be4f039"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;*&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXtrueEE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:kDataType.kUnsigned.true:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01kDataType_00_01kUnsigned_00_01true_01_4_1a1d99943821b4bdd36c1aec9f911ee992"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;kDataType, kUnsigned, true&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE">
<span id="_CPPv3I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE"></span><span id="_CPPv2I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kUnsigned</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kBOOL_00_01kUnsigned_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kBOOL</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kBOOL, kUnsigned&gt;::kUnsigned"><span class="n"><span class="pre">kUnsigned</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kBOOL_00_01kUnsigned_01_4_1a3ef7ec1c22f6cc0b397ab5eb28c232eb"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kBOOL.kUnsigned:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kBOOL_00_01kUnsigned_01_4_1a60ce7e6d8364224764d2415e4b709925"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;bool&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kBOOL.kUnsigned:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kBOOL_00_01kUnsigned_01_4_1aea0d24ee412e727b1e7d1831501600df"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kBOOL, kUnsigned&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kFLOAT_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kFLOAT</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kFLOAT_01_4_1af6ef0c2bf073670006e1a68fd861495b"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="kt"><span class="pre">float</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kFLOAT:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kFLOAT_01_4_1a7253109d1e4c01ed33157460dba35ec7"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;float&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kFLOAT:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kFLOAT_01_4_1afff1a8fde1dc21c665c8e1f3f8cc3fbd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kFLOAT&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kHALF_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kHALF</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kHALF_01_4_1a08c9427d16066ff8ee9e9deb35a91871"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">half</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kHALF:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kHALF_01_4_1a661db84e273578749fbd43d1f6829aa0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;half&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kHALF:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kHALF_01_4_1aad4af9eb10a14553b0dc46bd210454c1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kHALF&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT32</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_01_4_1a37fa1b65a6bece3eb638f7693132053c"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT32:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_01_4_1a98f4973f56a92c96b5bd1c4298b4ca72"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;int32&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT32:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_01_4_1ad02b1cf34aed9754967e7bb76a55fb0b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kINT32&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXtrueEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_00_01true_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT32</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXtrueEE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_00_01true_01_4_1a41a0f3bcdb82b843a9594fbe518c148d"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint32_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXtrueEE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT32.true:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_00_01true_01_4_1a455a075a88a89b0bf29b5ac1afc54320"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;uint32&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXtrueEE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT32.true:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT32_00_01true_01_4_1a420375ac7c713ee26156c7e143327393"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kINT32, true&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT64</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_01_4_1a288b116cbff3532cf2b80e7206277846"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT64:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_01_4_1a912e86b1f6bb4572292b27ed87819384"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;int64&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT64:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_01_4_1ac9994f6271fd3dd86e3be01df67a5577"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kINT64&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXtrueEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_00_01true_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT64</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXtrueEE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_00_01true_01_4_1a8376cfb6029bfec1e7aaa9c29369b094"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXtrueEE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT64.true:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_00_01true_01_4_1ae082c5ad1c2c409b8cbedc64b69e7f1d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;uint64&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXtrueEE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT64.true:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT64_00_01true_01_4_1af5140fb67b9b5cae4c40ea6f998f1223"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kINT64, true&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT8_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT8</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT8_01_4_1a51b8ee14d86b4b2a6cc5e57a1d524c43"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT8:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT8_01_4_1ad45ffa244d70cff61abcc7c89436e839"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;int8&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kINT8:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kINT8_01_4_1ad29e96b3fe577dba8524735a4f42f768"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kINT8&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE">
<span id="_CPPv3I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE"></span><span id="_CPPv2I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">kUnsigned</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kUINT8_00_01kUnsigned_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataTypeTraits</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kUINT8</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kUINT8, kUnsigned&gt;::kUnsigned"><span class="n"><span class="pre">kUnsigned</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kUINT8_00_01kUnsigned_01_4_1a5a3089fdad68ca4ce8bf36137d18a147"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">type</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4nameE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4nameE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4nameE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kUINT8.kUnsigned:::name__cA"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kUINT8_00_01kUnsigned_01_4_1a997c0219602be3dfcde787c95462e1d5"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">name</span></span></span><span class="p"><span class="pre">[</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;uint8&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4nameE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4sizeE">
<span id="_CPPv3N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4sizeE"></span><span id="_CPPv2N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4sizeE"></span><span id="tensorrt_llm::runtime::DataTypeTraits:nvinfer1::DataType::kUINT8.kUnsigned:::size__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1DataTypeTraits_3_01nvinfer1_1_1DataType_1_1kUINT8_00_01kUnsigned_01_4_1abb3870f3b3387d0cfedb4af98f5d71a6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">size</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">sizeof</span></span><span class="p"><span class="pre">(</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE" title="tensorrt_llm::runtime::DataTypeTraits&lt;nvinfer1::DataType::kUINT8, kUnsigned&gt;::type"><span class="n"><span class="pre">type</span></span></a><span class="p"><span class="pre">)</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4sizeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp class">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBufferE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBufferE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBufferE"></span><span id="tensorrt_llm::runtime::IBuffer"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer"></span><span class="k"><span class="pre">class</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">IBuffer</span></span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="Link to this definition">#</a><br /></dt>
<dd><p>Subclassed by <a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1ITensor"><span class="std std-ref">tensorrt_llm::runtime::ITensor</span></a></p>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-types">Public Types</p>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer9UniquePtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer9UniquePtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ab5684b52d1990c0a0f70c25d0b6c1465"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UniquePtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer9SharedPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer9SharedPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a2f0c3696d87a3887b38da126cbef1759"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SharedPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a3892842dc7c9dcc1cad622470ac97999"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">UniqueConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">unique_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a8eefcf151b57310d8ca78b8e25c46c39"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">SharedConstPtr</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">shared_ptr</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp type">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer8DataTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer8DataTypeE"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a4036935d2bbb442bf9d071df2eff24bd"></span><span class="k"><span class="pre">using</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">DataType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-functions">Public Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4dataEv">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4dataEv"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4dataEv"></span><span id="tensorrt_llm::runtime::IBuffer::data"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a56ee6dcc7c3b905e6b929e3e56041874"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">data</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4dataEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a pointer to underlying array. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer4dataEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer4dataEv"></span><span id="tensorrt_llm::runtime::IBuffer::dataC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ab18c55dc858ab44df1216f9f1bce157c"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">data</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a pointer to underlying array. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::data__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a6f57afc5472f47ec5c76ce5cddb16849"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">data</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">index</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a pointer to the underlying array at a given element index. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::data__std::sC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ab59d0db536d7cb31593cec33e588fc43"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">data</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">index</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a pointer to the underlying array at a given element index. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer7getSizeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer7getSizeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer7getSizeEv"></span><span id="tensorrt_llm::runtime::IBuffer::getSizeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ab280c88977e5cf7d3c3ab88e29dcbe62"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSize</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer7getSizeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the size (in number of elements) of the buffer. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer14getSizeInBytesEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer14getSizeInBytesEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer14getSizeInBytesEv"></span><span id="tensorrt_llm::runtime::IBuffer::getSizeInBytesC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a3121c0295ee572bb45067bba2d35defa"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getSizeInBytes</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer14getSizeInBytesEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the size (in bytes) of the buffer. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer11getCapacityEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer11getCapacityEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer11getCapacityEv"></span><span id="tensorrt_llm::runtime::IBuffer::getCapacityC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a33c48c142ee4ee1c1537d1bef5a7ed16"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getCapacity</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer11getCapacityEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the capacity of the buffer. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer11getDataTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer11getDataTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer11getDataTypeEv"></span><span id="tensorrt_llm::runtime::IBuffer::getDataTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a006dc512c6b37d582a2e825249c4a3a2"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE" title="tensorrt_llm::runtime::IBuffer::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getDataType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer11getDataTypeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the data type of the buffer. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer15getDataTypeNameEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer15getDataTypeNameEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer15getDataTypeNameEv"></span><span id="tensorrt_llm::runtime::IBuffer::getDataTypeNameC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a8aa7843674fa7d71b1c0e894a312bd94"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDataTypeName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer15getDataTypeNameEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer13getMemoryTypeEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer13getMemoryTypeEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer13getMemoryTypeEv"></span><span id="tensorrt_llm::runtime::IBuffer::getMemoryTypeC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ade5fc6e89a07ad03927a616fdcbb0463"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">getMemoryType</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer13getMemoryTypeEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns the memory type of the buffer. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer17getMemoryTypeNameEv">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer17getMemoryTypeNameEv"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer17getMemoryTypeNameEv"></span><span id="tensorrt_llm::runtime::IBuffer::getMemoryTypeNameC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a9108d1411aea38779b0aa4234250c9f5"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getMemoryTypeName</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer17getMemoryTypeNameEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer6resizeENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer6resizeENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer6resizeENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::resize__std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1af1281049c93d0a1d61b8682170761ad4"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">resize</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">newSize</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer6resizeENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Resizes the buffer. This is a no-op if the new size is smaller than or equal to the current capacity. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer7releaseEv">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer7releaseEv"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer7releaseEv"></span><span id="tensorrt_llm::runtime::IBuffer::release"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a51dc4a186d9b315dfe77aacd33677ff7"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">release</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7releaseEv" title="Link to this definition">#</a><br /></dt>
<dd><p>Releases the buffer. It will be reset to nullptr. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBufferD0Ev">
<span id="_CPPv3N12tensorrt_llm7runtime7IBufferD0Ev"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBufferD0Ev"></span><span id="tensorrt_llm::runtime::IBuffer::~IBuffer"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1abeb460077884adc9aec0c351c9ef3637"></span><span class="k"><span class="pre">virtual</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">~IBuffer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBufferD0Ev" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer"></span><span id="tensorrt_llm::runtime::IBuffer::IBuffer__IBufferCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ac9fd5dfcd8486d8f8150fdb470c38055"></span><span class="sig-name descname"><span class="n"><span class="pre">IBuffer</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer" title="tensorrt_llm::runtime::IBuffer::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Not allowed to copy. </p>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBufferaSERK7IBuffer">
<span id="_CPPv3N12tensorrt_llm7runtime7IBufferaSERK7IBuffer"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBufferaSERK7IBuffer"></span><span id="tensorrt_llm::runtime::IBuffer::assign-operator__IBufferCR"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ac2d5c67a705ab550d6aaecd01108408e"></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="sig-name descname"><span class="k"><span class="pre">operator</span></span><span class="o"><span class="pre">=</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE" title="tensorrt_llm::runtime::IBuffer"><span class="n"><span class="pre">IBuffer</span></span></a><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="p"><span class="pre">&amp;</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">delete</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBufferaSERK7IBuffer" title="Link to this definition">#</a><br /></dt>
<dd><p>Not allowed to copy. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-functions">Public Static Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer15getDataTypeNameE8DataType">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer15getDataTypeNameE8DataType"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer15getDataTypeNameE8DataType"></span><span id="tensorrt_llm::runtime::IBuffer::getDataTypeName__DataType"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1afda51ea16de70b983603ca5e6225e255"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">char</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="sig-name descname"><span class="n"><span class="pre">getDataTypeName</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE" title="tensorrt_llm::runtime::IBuffer::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">dataType</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer15getDataTypeNameE8DataType" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::slice__SharedPtr.std::s.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a6b9aed7e66f2bfc1f3f710d12dbceea6"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">buffer</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Creates a sliced view on the underlying <code class="docutils literal notranslate"><span class="pre">buffer</span></code>. The view will have the same data type as <code class="docutils literal notranslate"><span class="pre">buffer</span></code>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>buffer</strong> – The buffer to view. </p></li>
<li><p><strong>offset</strong> – The offset of the view. </p></li>
<li><p><strong>size</strong> – The size of the view. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">buffer</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceERR9TConstPtrNSt6size_tENSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceERR9TConstPtrNSt6size_tENSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="tensorrt_llm::runtime::IBuffer::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a40f5f57b98c9918b617c1de832390b9c"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE" title="tensorrt_llm::runtime::IBuffer::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="tensorrt_llm::runtime::IBuffer::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::slice__SharedPtr.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a310d420f890b10b16951eae4b0227d21"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">buffer</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceERR9TConstPtrNSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceERR9TConstPtrNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::IBuffer::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a9ffdc0f354482f983b6a16825ad25109"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE" title="tensorrt_llm::runtime::IBuffer::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">slice</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::IBuffer::slice::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">offset</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtr">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtr"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtr"></span><span id="tensorrt_llm::runtime::IBuffer::view__SharedPtr"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a4fb6a8d4a92376cdea0957a26629f53b"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtr" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a view on the underlying <code class="docutils literal notranslate"><span class="pre">tensor</span></code> which can be independently resized. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><p><strong>tensor</strong> – The tensor to view. </p>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtrNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtrNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtrNSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::view__SharedPtr.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a58bc0fcaa0356cf9e20ce1d06b16c70f"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE" title="tensorrt_llm::runtime::IBuffer::SharedPtr"><span class="n"><span class="pre">SharedPtr</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">tensor</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns a view on the underlying <code class="docutils literal notranslate"><span class="pre">tensor</span></code> with a different size. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>tensor</strong> – The tensor to view. </p></li>
<li><p><strong>size</strong> – The size of the view. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>A view on the <code class="docutils literal notranslate"><span class="pre">tensor</span></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewE14UniqueConstPtrRR9TConstPtrNSt6size_tE">
<span id="_CPPv3I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewERR9TConstPtrNSt6size_tE"></span><span id="_CPPv2I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewERR9TConstPtrNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">TConstPtr</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">enable_if_t</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">is_const_v</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE" title="tensorrt_llm::runtime::PointerElementType"><span class="n"><span class="pre">PointerElementType</span></span></a><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::IBuffer::view::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">int</span></span><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="m"><span class="pre">0</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1af28ec3097ffc66614052cef9392265bb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE" title="tensorrt_llm::runtime::IBuffer::UniqueConstPtr"><span class="n"><span class="pre">UniqueConstPtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">view</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="tensorrt_llm::runtime::IBuffer::view::TConstPtr"><span class="n"><span class="pre">TConstPtr</span></span></a><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">tensor</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewE14UniqueConstPtrRR9TConstPtrNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tENSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tENSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::wrap__voidP.DataType.std::s.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a6cf1157eb353c90dbe12711be9af63f2"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE" title="tensorrt_llm::runtime::IBuffer::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">capacity</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Wraps the given <code class="docutils literal notranslate"><span class="pre">data</span></code> in an <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code>. The <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code> will not own the underlying <code class="docutils literal notranslate"><span class="pre">data</span></code> and cannot be resized beyond <code class="docutils literal notranslate"><span class="pre">capacity</span></code>. </p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>data</strong> – The data to wrap. </p></li>
<li><p><strong>type</strong> – The data type of the <code class="docutils literal notranslate"><span class="pre">data</span></code>. </p></li>
<li><p><strong>size</strong> – The size of the buffer. </p></li>
<li><p><strong>capacity</strong> – The capacity of the buffer. </p></li>
</ul>
</dd>
<dt class="field-even">Returns<span class="colon">:</span></dt>
<dd class="field-even"><p>An <code class="docutils literal notranslate"><a class="reference internal" href="#classtensorrt__llm_1_1runtime_1_1IBuffer"><span class="std std-ref"><span class="pre">IBuffer</span></span></a></code>. </p>
</dd>
</dl>
</dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tE">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tE"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::wrap__voidP.DataType.std::s"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1af1fb9e5497bfc31b149ff9477161068d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE" title="tensorrt_llm::runtime::IBuffer::DataType"><span class="n"><span class="pre">DataType</span></span></a><span class="w"> </span><span class="n sig-param"><span class="pre">type</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tENSt6size_tE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7IBuffer4wrapEP1TNSt6size_tENSt6size_tE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7IBuffer4wrapEP1TNSt6size_tENSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a1f5dd20e613af0bc2a05f0b09343535d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tENSt6size_tE" title="tensorrt_llm::runtime::IBuffer::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">capacity</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7IBuffer4wrapEP1TNSt6size_tE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7IBuffer4wrapEP1TNSt6size_tE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a2e4240d0d7f78a278716c8faccf5e9f9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tE" title="tensorrt_llm::runtime::IBuffer::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em>,</dd>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrRNSt6vectorI1TEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime7IBuffer4wrapERNSt6vectorI1TEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime7IBuffer4wrapERNSt6vectorI1TEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a84fdf2b484eee3440646edfece5b85b0"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE" title="tensorrt_llm::runtime::IBuffer::UniquePtr"><span class="n"><span class="pre">UniquePtr</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">wrap</span></span></span><span class="sig-paren">(</span>
<dl>
<dd><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">vector</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrRNSt6vectorI1TEE" title="tensorrt_llm::runtime::IBuffer::wrap::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="w"> </span><span class="p"><span class="pre">&amp;</span></span><span class="n sig-param"><span class="pre">v</span></span></em>,</dd>
</dl>
<span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrRNSt6vectorI1TEE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer10memoryTypeEPKv">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer10memoryTypeEPKv"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer10memoryTypeEPKv"></span><span id="tensorrt_llm::runtime::IBuffer::memoryType__voidCP"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1a2c714d66d0336cac0708008e59dd71cd"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">memoryType</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="kt"><span class="pre">void</span></span><span class="w"> </span><span class="k"><span class="pre">const</span></span><span class="w"> </span><span class="p"><span class="pre">*</span></span><span class="n sig-param"><span class="pre">data</span></span></em><span class="sig-paren">)</span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer10memoryTypeEPKv" title="Link to this definition">#</a><br /></dt>
<dd><p>Determine the memory type of a pointer. </p>
</dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-protected-functions">Protected Functions</p>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferEv">
<span id="_CPPv3N12tensorrt_llm7runtime7IBuffer7IBufferEv"></span><span id="_CPPv2N12tensorrt_llm7runtime7IBuffer7IBufferEv"></span><span id="tensorrt_llm::runtime::IBuffer::IBuffer"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1ad6ad0acb873c47b64fc8fec029696ac2"></span><span class="sig-name descname"><span class="n"><span class="pre">IBuffer</span></span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">default</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferEv" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp function">
<dt class="sig sig-object cpp" id="_CPPv4NK12tensorrt_llm7runtime7IBuffer7toBytesENSt6size_tE">
<span id="_CPPv3NK12tensorrt_llm7runtime7IBuffer7toBytesENSt6size_tE"></span><span id="_CPPv2NK12tensorrt_llm7runtime7IBuffer7toBytesENSt6size_tE"></span><span id="tensorrt_llm::runtime::IBuffer::toBytes__std::sC"></span><span class="target" id="classtensorrt__llm_1_1runtime_1_1IBuffer_1aade9d63190ac92fdb3561a61af5ab2ee"></span><span class="k"><span class="pre">inline</span></span><span class="w"> </span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">toBytes</span></span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">size_t</span></span><span class="w"> </span><span class="n sig-param"><span class="pre">size</span></span></em><span class="sig-paren">)</span><span class="w"> </span><span class="k"><span class="pre">const</span></span><a class="headerlink" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer7toBytesENSt6size_tE" title="Link to this definition">#</a><br /></dt>
<dd><p>Returns an array index or size in bytes. </p>
</dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime16MemoryTypeStringE">
<span id="_CPPv3I_10MemoryTypeEN12tensorrt_llm7runtime16MemoryTypeStringE"></span><span id="_CPPv2I_10MemoryTypeEN12tensorrt_llm7runtime16MemoryTypeStringE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><a class="headerlink" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime16MemoryTypeStringE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kCPU_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kCPUE" title="tensorrt_llm::runtime::MemoryType::kCPU"><span class="n"><span class="pre">kCPU</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEE5valueE"></span><span id="tensorrt_llm::runtime::MemoryTypeString:MemoryType::kCPU:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kCPU_01_4_1af3b289e3cfd246c2b1ee25aec5eeb6e1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;CPU&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kGPU_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kGPUE" title="tensorrt_llm::runtime::MemoryType::kGPU"><span class="n"><span class="pre">kGPU</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEE5valueE"></span><span id="tensorrt_llm::runtime::MemoryTypeString:MemoryType::kGPU:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kGPU_01_4_1a0fc1c309e17dd52d719cafddbcc6cb12"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;GPU&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kPINNED_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType7kPINNEDE" title="tensorrt_llm::runtime::MemoryType::kPINNED"><span class="n"><span class="pre">kPINNED</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEE5valueE"></span><span id="tensorrt_llm::runtime::MemoryTypeString:MemoryType::kPINNED:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kPINNED_01_4_1a5e877eb90ca7cbd5b5269d4f092e69e4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;PINNED&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kPINNEDPOOL_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE" title="tensorrt_llm::runtime::MemoryType::kPINNEDPOOL"><span class="n"><span class="pre">kPINNEDPOOL</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEE5valueE"></span><span id="tensorrt_llm::runtime::MemoryTypeString:MemoryType::kPINNEDPOOL:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kPINNEDPOOL_01_4_1aad3eee9ea444088033995a877a00e9c9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;PINNEDPOOL&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kUVM_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">MemoryTypeString</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE" title="tensorrt_llm::runtime::MemoryType"><span class="n"><span class="pre">MemoryType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kUVME" title="tensorrt_llm::runtime::MemoryType::kUVM"><span class="n"><span class="pre">kUVM</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEE5valueE"></span><span id="tensorrt_llm::runtime::MemoryTypeString:MemoryType::kUVM:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1MemoryTypeString_3_01MemoryType_1_1kUVM_01_4_1a743212e7128317c3b6a0af4ad619afa9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="s"><span class="pre">&quot;UVM&quot;</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I0_bEN12tensorrt_llm7runtime11TRTDataTypeE">
<span id="_CPPv3I0_bEN12tensorrt_llm7runtime11TRTDataTypeE"></span><span id="_CPPv2I0_bEN12tensorrt_llm7runtime11TRTDataTypeE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="kt"><span class="pre">bool</span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><a class="headerlink" href="#_CPPv4I0_bEN12tensorrt_llm7runtime11TRTDataTypeE" title="Link to this definition">#</a><br /></dt>
<dd><div class="docutils container">
<em>#include &lt;iBuffer.h&gt;</em></div>
<p>For converting a C++ data type to a TensorRT data type. </p>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIbEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIbEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIbEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01bool_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">bool</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIbEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIbE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIbE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIbE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:b:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01bool_01_4_1a590c40a56a5eabf6069340edc07d84e4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kBOOL</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIbE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIfEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIfEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIfEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01float_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">float</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIfEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIfE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIfE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIfE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:float:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01float_01_4_1a0a72c60c5a9b946e64a960d1075a5af9"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kFLOAT</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIfE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeI4halfEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeI4halfEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeI4halfEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01half_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">half</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeI4halfEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeI4halfE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeI4halfE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeI4halfE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:half:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01half_01_4_1a2febf53869fcfd71830c674652e7c7d4"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kHALF</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeI4halfE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01kernels_1_1FinishedState_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">kernels</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">FinishedState</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:kernels::FinishedState:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01kernels_1_1FinishedState_01_4_1af0e2893266f9a1af9a97644162b45afa"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">TRTDataType</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">kernels</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">FinishedState</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">UnderlyingType</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">value</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01kernels_1_1KVCacheIndex_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">kernels</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">KVCacheIndex</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:kernels::KVCacheIndex:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01kernels_1_1KVCacheIndex_01_4_1a8a9684f587b9b6ac46ac769f6f205419"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">TRTDataType</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">kernels</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">KVCacheIndex</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">UnderlyingType</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">value</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01runtime_1_1RequestType_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11RequestTypeE" title="tensorrt_llm::runtime::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:runtime::RequestType:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01runtime_1_1RequestType_01_4_1a689d61d98e3959c3f520274718c23541"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="n"><span class="pre">TRTDataType</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">underlying_type_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="executor.html#_CPPv4N12tensorrt_llm7runtimeE" title="tensorrt_llm::runtime"><span class="n"><span class="pre">runtime</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11RequestTypeE" title="tensorrt_llm::runtime::RequestType"><span class="n"><span class="pre">RequestType</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">value</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int32__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int32_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::int32_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int32__t_01_4_1a10403a956a1d979e0cd10c8034e9f3e1"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT32</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int64__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int64_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::int64_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int64__t_01_4_1af2839adf78453826b3a7f8422c9130fb"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT64</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int8__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">int8_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::int8_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1int8__t_01_4_1a996ff7940394a814ae164728312bf5ff"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint32__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint32_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::uint32_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint32__t_01_4_1a96b97cf0edb4ad984dd69278acfabd10"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="tensorrt_llm::runtime::BufferDataType"><span class="n"><span class="pre">BufferDataType</span></span></a><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT32</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint64__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint64_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::uint64_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint64__t_01_4_1a6d279398fbfedbbfffd0f7fbd1eb80b8"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="tensorrt_llm::runtime::BufferDataType"><span class="n"><span class="pre">BufferDataType</span></span></a><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kINT64</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint8__t_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">uint8_t</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:std::uint8_t:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01std_1_1uint8__t_01_4_1a60d418bc6cea9f845ee353d78f0c3d66"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv48nvinfer1" title="nvinfer1"><span class="n"><span class="pre">nvinfer1</span></span></a><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">DataType</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">kUINT8</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE">
<span id="_CPPv3I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE"></span><span id="_CPPv2I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="k"><span class="pre">typename</span></span><span class="w"> </span><span class="sig-name descname sig-name-template"><span class="n"><span class="pre">T</span></span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01T_01_5_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE" title="tensorrt_llm::runtime::TRTDataType&lt;T*&gt;::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIP1TE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIP1TE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:TP:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01T_01_5_01_4_1a6e605b21926aa5997834a5ed9069610d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="tensorrt_llm::runtime::BufferDataType"><span class="n"><span class="pre">BufferDataType</span></span></a><span class="p"><span class="pre">{</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE" title="tensorrt_llm::runtime::TRTDataType&lt;T*&gt;::kUnderlyingType"><span class="n"><span class="pre">kUnderlyingType</span></span></a><span class="p"><span class="pre">.</span></span><span class="n"><span class="pre">getDataType</span></span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE" title="tensorrt_llm::runtime::TRTDataType&lt;T*&gt;::kUnderlyingType"><span class="n"><span class="pre">kUnderlyingType</span></span></a><span class="p"><span class="pre">.</span></span><span class="n"><span class="pre">isUnsigned</span></span><span class="p"><span class="pre">(</span></span><span class="p"><span class="pre">)</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">true</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
<div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-private-static-attributes">Private Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE"></span><span id="tensorrt_llm::runtime::TRTDataType:TP:::kUnderlyingType__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01T_01_5_01_4_1aa44a2359868876e32c25fdab6f9faf2d"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">kUnderlyingType</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="tensorrt_llm::runtime::BufferDataType"><span class="n"><span class="pre">BufferDataType</span></span></a><span class="p"><span class="pre">{</span></span><span class="n"><span class="pre">TRTDataType</span></span><span class="p"><span class="pre">&lt;</span></span><span class="n"><span class="pre">std</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">remove_const_t</span></span><span class="p"><span class="pre">&lt;</span></span><a class="reference internal" href="#_CPPv4I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE" title="tensorrt_llm::runtime::TRTDataType&lt;T*&gt;::T"><span class="n"><span class="pre">T</span></span></a><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="k"><span class="pre">false</span></span><span class="p"><span class="pre">&gt;</span></span><span class="p"><span class="pre">::</span></span><span class="n"><span class="pre">value</span></span><span class="p"><span class="pre">}</span></span><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
<dl class="cpp struct">
<dt class="sig sig-object cpp" id="_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIPvEE">
<span id="_CPPv3IEN12tensorrt_llm7runtime11TRTDataTypeIPvEE"></span><span id="_CPPv2IEN12tensorrt_llm7runtime11TRTDataTypeIPvEE"></span><span class="k"><span class="pre">template</span></span><span class="p"><span class="pre">&lt;</span></span><span class="p"><span class="pre">&gt;</span></span><br /><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01void_01_5_01_4"></span><span class="k"><span class="pre">struct</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">TRTDataType</span></span></span><span class="p"><span class="pre">&lt;</span></span><span class="kt"><span class="pre">void</span></span><span class="p"><span class="pre">*</span></span><span class="p"><span class="pre">&gt;</span></span><a class="headerlink" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIPvEE" title="Link to this definition">#</a><br /></dt>
<dd><div class="breathe-sectiondef docutils container">
<p class="breathe-sectiondef-title rubric" id="breathe-section-title-public-static-attributes">Public Static Attributes</p>
<dl class="cpp var">
<dt class="sig sig-object cpp" id="_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIPvE5valueE">
<span id="_CPPv3N12tensorrt_llm7runtime11TRTDataTypeIPvE5valueE"></span><span id="_CPPv2N12tensorrt_llm7runtime11TRTDataTypeIPvE5valueE"></span><span id="tensorrt_llm::runtime::TRTDataType:voidP:::value__auto"></span><span class="target" id="structtensorrt__llm_1_1runtime_1_1TRTDataType_3_01void_01_5_01_4_1a63d38011d7c02b6ce399d457537b1a93"></span><span class="k"><span class="pre">static</span></span><span class="w"> </span><span class="k"><span class="pre">constexpr</span></span><span class="w"> </span><span class="kt"><span class="pre">auto</span></span><span class="w"> </span><span class="sig-name descname"><span class="n"><span class="pre">value</span></span></span><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE" title="tensorrt_llm::runtime::BufferDataType"><span class="n"><span class="pre">BufferDataType</span></span></a><span class="p"><span class="pre">::</span></span><a class="reference internal" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE" title="tensorrt_llm::runtime::BufferDataType::kTrtPointerType"><span class="n"><span class="pre">kTrtPointerType</span></span></a><a class="headerlink" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIPvE5valueE" title="Link to this definition">#</a><br /></dt>
<dd></dd></dl>
</div>
</dd></dl>
</dd></dl>
</dd></dl>
</section>
</section>
</article>
<footer class="prev-next-footer d-print-none">
<div class="prev-next-area">
<a class="left-prev"
href="executor.html"
title="previous page">
<i class="fa-solid fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">Executor</p>
</div>
</a>
<a class="right-next"
href="../commands/trtllm-build.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">trtllm-build</p>
</div>
<i class="fa-solid fa-angle-right"></i>
</a>
</div>
</footer>
</div>
<dialog id="pst-secondary-sidebar-modal"></dialog>
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
<div class="sidebar-secondary-item">
<div
id="pst-page-navigation-heading-2"
class="page-toc tocsection onthispage">
<i class="fa-solid fa-list"></i> On this page
</div>
<nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#lookaheadmodule-h">lookaheadModule.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv412tensorrt_llm"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModuleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LookaheadModule</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">LookaheadModule()</span></code></a></li>
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule15LookaheadModuleEv"><code class="docutils literal notranslate"><span class="pre">LookaheadModule()</span></code></a></li>
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule18setExecutionConfigERKN8executor23LookaheadDecodingConfigE"><code class="docutils literal notranslate"><span class="pre">setExecutionConfig()</span></code></a></li>
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime15LookaheadModule18getExecutionConfigEv"><code class="docutils literal notranslate"><span class="pre">getExecutionConfig()</span></code></a></li>
<li class="toc-h6 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15LookaheadModule16mExecutionConfigE"><code class="docutils literal notranslate"><span class="pre">mExecutionConfig</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#decoderstate-h">decoderState.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoderE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder::BeamSearchBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers17BeamSearchBuffersERK13BufferManager"><code class="docutils literal notranslate"><span class="pre">BeamSearchBuffers()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7reshapeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers21mOutputBeamHypothesesE"><code class="docutils literal notranslate"><span class="pre">mOutputBeamHypotheses</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers15mCumLogProbsTmpE"><code class="docutils literal notranslate"><span class="pre">mCumLogProbsTmp</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder17BeamSearchBuffers7mNumSMsE"><code class="docutils literal notranslate"><span class="pre">mNumSMs</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderStateE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder::DecoderState</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13LlmRequestPtrE"><code class="docutils literal notranslate"><span class="pre">LlmRequestPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13RequestVectorE"><code class="docutils literal notranslate"><span class="pre">RequestVector</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16DecodingInputPtrE"><code class="docutils literal notranslate"><span class="pre">DecodingInputPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState17DecodingOutputPtrE"><code class="docutils literal notranslate"><span class="pre">DecodingOutputPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState12DecoderStateEN8nvinfer18DataTypeERK13BufferManager"><code class="docutils literal notranslate"><span class="pre">DecoderState()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState34allocateSpeculativeDecodingBuffersE23SpeculativeDecodingModeN8nvinfer18DataTypeERK13BufferManager"><code class="docutils literal notranslate"><span class="pre">allocateSpeculativeDecodingBuffers()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState5setupE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"><code class="docutils literal notranslate"><span class="pre">setup()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24setupSpeculativeDecodingERK23SpeculativeDecodingMode10SizeType32RK11ModelConfigRK11WorldConfigRK13BufferManager"><code class="docutils literal notranslate"><span class="pre">setupSpeculativeDecoding()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24setupExplicitDraftTokensEN26ExplicitDraftTokensBuffers6InputsE"><code class="docutils literal notranslate"><span class="pre">setupExplicitDraftTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14setupLookaheadE24LookaheadDecodingBuffers"><code class="docutils literal notranslate"><span class="pre">setupLookahead()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState10setupEagleEN12EagleBuffers6InputsE"><code class="docutils literal notranslate"><span class="pre">setupEagle()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState16disableLookaheadERK13RequestVector"><code class="docutils literal notranslate"><span class="pre">disableLookahead()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getFinishedSumEv"><code class="docutils literal notranslate"><span class="pre">getFinishedSum()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishReasonsEv"><code class="docutils literal notranslate"><span class="pre">getFinishReasons()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsEv"><code class="docutils literal notranslate"><span class="pre">getIds()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState6getIdsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getIds()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsEv"><code class="docutils literal notranslate"><span class="pre">getGatheredIds()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getGatheredIdsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getGatheredIds()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState12getParentIdsEv"><code class="docutils literal notranslate"><span class="pre">getParentIds()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsEv"><code class="docutils literal notranslate"><span class="pre">getCumLogProbs()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState14getCumLogProbsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getCumLogProbs()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsEv"><code class="docutils literal notranslate"><span class="pre">getLogProbs()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState11getLogProbsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getLogProbs()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getSequenceLengthsEv"><code class="docutils literal notranslate"><span class="pre">getSequenceLengths()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getAllNewTokensEv"><code class="docutils literal notranslate"><span class="pre">getAllNewTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState18getNextDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">getNextDraftTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getPrevDraftTokensLengthsEv"><code class="docutils literal notranslate"><span class="pre">getPrevDraftTokensLengths()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState25getNextDraftTokensLengthsEv"><code class="docutils literal notranslate"><span class="pre">getNextDraftTokensLengths()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState24getAcceptedLengthsCumSumEv"><code class="docutils literal notranslate"><span class="pre">getAcceptedLengthsCumSum()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getAcceptedPackedPathsEv"><code class="docutils literal notranslate"><span class="pre">getAcceptedPackedPaths()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState16getFinishedStepsEv"><code class="docutils literal notranslate"><span class="pre">getFinishedSteps()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState15getMaxBeamWidthEv"><code class="docutils literal notranslate"><span class="pre">getMaxBeamWidth()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getMaxSequenceLengthEv"><code class="docutils literal notranslate"><span class="pre">getMaxSequenceLength()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState27getMaxDecodingDecoderTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingDecoderTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getMaxDecodingEngineTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingEngineTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensEv"><code class="docutils literal notranslate"><span class="pre">getNumDecodingEngineTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getNumDecodingEngineTokensE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getNumDecodingEngineTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState26setNumDecodingEngineTokensE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">setNumDecodingEngineTokens()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState26getSpeculativeDecodingModeEv"><code class="docutils literal notranslate"><span class="pre">getSpeculativeDecodingMode()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState20getBeamSearchBuffersEv"><code class="docutils literal notranslate"><span class="pre">getBeamSearchBuffers()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState21getJointDecodingInputEv"><code class="docutils literal notranslate"><span class="pre">getJointDecodingInput()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7decoder12DecoderState22getJointDecodingOutputEv"><code class="docutils literal notranslate"><span class="pre">getJointDecodingOutput()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBatchSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxBatchSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState13mMaxBeamWidthE"><code class="docutils literal notranslate"><span class="pre">mMaxBeamWidth</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mMaxSequenceLengthE"><code class="docutils literal notranslate"><span class="pre">mMaxSequenceLength</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState19mJointDecodingInputE"><code class="docutils literal notranslate"><span class="pre">mJointDecodingInput</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState20mJointDecodingOutputE"><code class="docutils literal notranslate"><span class="pre">mJointDecodingOutput</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState14mFinishedStepsE"><code class="docutils literal notranslate"><span class="pre">mFinishedSteps</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState18mBeamSearchBuffersE"><code class="docutils literal notranslate"><span class="pre">mBeamSearchBuffers</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState25mMaxDecodingDecoderTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxDecodingDecoderTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mMaxDecodingEngineTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxDecodingEngineTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mNumDecodingEngineTokensE"><code class="docutils literal notranslate"><span class="pre">mNumDecodingEngineTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7decoder12DecoderState24mSpeculativeDecodingModeE"><code class="docutils literal notranslate"><span class="pre">mSpeculativeDecodingMode</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#request-h">request.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batchE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder_batch</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7RequestE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder_batch::Request</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request14TensorConstPtrE"><code class="docutils literal notranslate"><span class="pre">TensorConstPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request9BufferPtrE"><code class="docutils literal notranslate"><span class="pre">BufferPtr</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request7RequestE14TensorConstPtr10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">Request()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request3idsE"><code class="docutils literal notranslate"><span class="pre">ids</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request8inputLenE"><code class="docutils literal notranslate"><span class="pre">inputLen</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12maxNewTokensE"><code class="docutils literal notranslate"><span class="pre">maxNewTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5endIdE"><code class="docutils literal notranslate"><span class="pre">endId</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request28generatedTokensPerEngineStepE"><code class="docutils literal notranslate"><span class="pre">generatedTokensPerEngineStep</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13embeddingBiasE"><code class="docutils literal notranslate"><span class="pre">embeddingBias</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request12badWordsListE"><code class="docutils literal notranslate"><span class="pre">badWordsList</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13stopWordsListE"><code class="docutils literal notranslate"><span class="pre">stopWordsList</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftTokensE"><code class="docutils literal notranslate"><span class="pre">draftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11draftLogitsE"><code class="docutils literal notranslate"><span class="pre">draftLogits</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11medusaPathsE"><code class="docutils literal notranslate"><span class="pre">medusaPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request13medusaTreeIdsE"><code class="docutils literal notranslate"><span class="pre">medusaTreeIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request5dtypeE"><code class="docutils literal notranslate"><span class="pre">dtype</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request22lookaheadRuntimeConfigE"><code class="docutils literal notranslate"><span class="pre">lookaheadRuntimeConfig</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch7Request11eagleConfigE"><code class="docutils literal notranslate"><span class="pre">eagleConfig</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loracache-h">loraCache.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9to_stringERKN9LoraCache21TaskLayerModuleConfigE"><code class="docutils literal notranslate"><span class="pre">to_string()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN9LoraCache21TaskLayerModuleConfigE"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCacheE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCache</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10TaskIdTypeE"><code class="docutils literal notranslate"><span class="pre">TaskIdType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache28TaskLayerModuleConfigListPtrE"><code class="docutils literal notranslate"><span class="pre">TaskLayerModuleConfigListPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9LoraCacheERK26LoraCachePageManagerConfigRK11ModelConfigRK11WorldConfigRK13BufferManager"><code class="docutils literal notranslate"><span class="pre">LoraCache()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache3putE10TaskIdType9TensorPtr9TensorPtrb"><code class="docutils literal notranslate"><span class="pre">put()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsE10TaskIdType9TensorPtr9TensorPtr"><code class="docutils literal notranslate"><span class="pre">loadWeights()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache8isLoadedE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">isLoaded()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache6isDoneE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">isDone()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache3hasE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">has()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache3getE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">get()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache4bumpE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">bump()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12markTaskDoneE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">markTaskDone()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11markAllDoneEv"><code class="docutils literal notranslate"><span class="pre">markAllDone()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">determineNumPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache17determineNumPagesE9TensorPtr"><code class="docutils literal notranslate"><span class="pre">determineNumPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache4fitsE9TensorPtr"><code class="docutils literal notranslate"><span class="pre">fits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache8copyTaskE10TaskIdTypeR9LoraCacheb"><code class="docutils literal notranslate"><span class="pre">copyTask()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache11getNumPagesEv"><code class="docutils literal notranslate"><span class="pre">getNumPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache10getPagePtrE6size_t"><code class="docutils literal notranslate"><span class="pre">getPagePtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11copyToPagesE9TensorPtr9TensorPtrRK11ModelConfigRK11WorldConfigNSt13unordered_mapI10SizeType3210LoraModuleEERK13BufferManagerRKNSt6vectorI9TensorPtrEERKNSt6vectorINSt6size_tEEE"><code class="docutils literal notranslate"><span class="pre">copyToPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17splitTransposeCpuER7ITensorRK7ITensor10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">splitTransposeCpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatusE"><code class="docutils literal notranslate"><span class="pre">ValueStatus</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus21kVALUE_STATUS_MISSINGE"><code class="docutils literal notranslate"><span class="pre">kVALUE_STATUS_MISSING</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus24kVALUE_STATUS_PROCESSINGE"><code class="docutils literal notranslate"><span class="pre">kVALUE_STATUS_PROCESSING</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11ValueStatus20kVALUE_STATUS_LOADEDE"><code class="docutils literal notranslate"><span class="pre">kVALUE_STATUS_LOADED</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12TaskValuePtrE"><code class="docutils literal notranslate"><span class="pre">TaskValuePtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11loadWeightsER9TaskValue9TensorPtr9TensorPtr"><code class="docutils literal notranslate"><span class="pre">loadWeights()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache18bumpTaskInProgressE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">bumpTaskInProgress()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache9getStatusE10TaskIdType"><code class="docutils literal notranslate"><span class="pre">getStatus()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache19claimPagesWithEvictE10SizeType32"><code class="docutils literal notranslate"><span class="pre">claimPagesWithEvict()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache16copyTaskMapPagesER9TaskValueRK9TaskValueRKNSt6vectorI6size_tEERK9LoraCache"><code class="docutils literal notranslate"><span class="pre">copyTaskMapPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache18mPageManagerConfigE"><code class="docutils literal notranslate"><span class="pre">mPageManagerConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12mModelConfigE"><code class="docutils literal notranslate"><span class="pre">mModelConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache12mWorldConfigE"><code class="docutils literal notranslate"><span class="pre">mWorldConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11mPagesMutexE"><code class="docutils literal notranslate"><span class="pre">mPagesMutex</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17mCachePageManagerE"><code class="docutils literal notranslate"><span class="pre">mCachePageManager</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache11mCacheMutexE"><code class="docutils literal notranslate"><span class="pre">mCacheMutex</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9mCacheMapE"><code class="docutils literal notranslate"><span class="pre">mCacheMap</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache16mInProgressTasksE"><code class="docutils literal notranslate"><span class="pre">mInProgressTasks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache10mDoneTasksE"><code class="docutils literal notranslate"><span class="pre">mDoneTasks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21mDeviceBufferManagersE"><code class="docutils literal notranslate"><span class="pre">mDeviceBufferManagers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache14mBufferManagerE"><code class="docutils literal notranslate"><span class="pre">mBufferManager</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache17mModuleIdToModuleE"><code class="docutils literal notranslate"><span class="pre">mModuleIdToModule</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime9LoraCache22splitTransposeCpuInnerEvR7ITensorRK7ITensor10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">splitTransposeCpuInner()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCache::TaskLayerModuleConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfigeqERKN9LoraCache21TaskLayerModuleConfigE"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6pageIdE"><code class="docutils literal notranslate"><span class="pre">pageId</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7slotIdxE"><code class="docutils literal notranslate"><span class="pre">slotIdx</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig6inSizeE"><code class="docutils literal notranslate"><span class="pre">inSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7outSizeE"><code class="docutils literal notranslate"><span class="pre">outSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8moduleIdE"><code class="docutils literal notranslate"><span class="pre">moduleId</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig7layerIdE"><code class="docutils literal notranslate"><span class="pre">layerId</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig11adapterSizeE"><code class="docutils literal notranslate"><span class="pre">adapterSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig8numSlotsE"><code class="docutils literal notranslate"><span class="pre">numSlots</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig16weightsInPointerE"><code class="docutils literal notranslate"><span class="pre">weightsInPointer</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17weightsOutPointerE"><code class="docutils literal notranslate"><span class="pre">weightsOutPointer</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache21TaskLayerModuleConfig17scalingVecPointerE"><code class="docutils literal notranslate"><span class="pre">scalingVecPointer</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCache::TaskValue</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueEv"><code class="docutils literal notranslate"><span class="pre">TaskValue()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueD0Ev"><code class="docutils literal notranslate"><span class="pre">~TaskValue()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERKNSt6vectorINSt6size_tEEERK28TaskLayerModuleConfigListPtrNSt4listI10TaskIdTypeE8iteratorEbbbb"><code class="docutils literal notranslate"><span class="pre">TaskValue()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue9TaskValueERR9TaskValue"><code class="docutils literal notranslate"><span class="pre">TaskValue()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValueaSERR9TaskValue"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7pageIdsE"><code class="docutils literal notranslate"><span class="pre">pageIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue7configsE"><code class="docutils literal notranslate"><span class="pre">configs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue2itE"><code class="docutils literal notranslate"><span class="pre">it</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue10inProgressE"><code class="docutils literal notranslate"><span class="pre">inProgress</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue6loadedE"><code class="docutils literal notranslate"><span class="pre">loaded</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue4doneE"><code class="docutils literal notranslate"><span class="pre">done</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9LoraCache9TaskValue14loadInProgressE"><code class="docutils literal notranslate"><span class="pre">loadInProgress</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCacheFullException</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullException22LoraCacheFullExceptionERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">LoraCacheFullException()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime22LoraCacheFullExceptionD0Ev"><code class="docutils literal notranslate"><span class="pre">~LoraCacheFullException()</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManagerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCachePageManager</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager20LoraCachePageManagerERK26LoraCachePageManagerConfigRK13BufferManager"><code class="docutils literal notranslate"><span class="pre">LoraCachePageManager()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10claimPagesE10SizeType32"><code class="docutils literal notranslate"><span class="pre">claimPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager17numAvailablePagesEv"><code class="docutils literal notranslate"><span class="pre">numAvailablePages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12releasePagesERKNSt6vectorINSt6size_tEEE"><code class="docutils literal notranslate"><span class="pre">releasePages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager8blockPtrE10SizeType32"><code class="docutils literal notranslate"><span class="pre">blockPtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20LoraCachePageManager7pagePtrENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">pagePtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager14mutablePagePtrENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">mutablePagePtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager10initializeERK13BufferManager"><code class="docutils literal notranslate"><span class="pre">initialize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mPageBlocksE"><code class="docutils literal notranslate"><span class="pre">mPageBlocks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager12mFreePageIdsE"><code class="docutils literal notranslate"><span class="pre">mFreePageIds</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager11mIsPageFreeE"><code class="docutils literal notranslate"><span class="pre">mIsPageFree</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20LoraCachePageManager7mConfigE"><code class="docutils literal notranslate"><span class="pre">mConfig</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraExpectedException</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedException21LoraExpectedExceptionERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">LoraExpectedException()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime21LoraExpectedExceptionD0Ev"><code class="docutils literal notranslate"><span class="pre">~LoraExpectedException()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#buffermanager-h">bufferManager.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::BufferManager</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10IBufferPtrE"><code class="docutils literal notranslate"><span class="pre">IBufferPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10ITensorPtrE"><code class="docutils literal notranslate"><span class="pre">ITensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">CudaStreamPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager14CudaMemPoolPtrE"><code class="docutils literal notranslate"><span class="pre">CudaMemPoolPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager13BufferManagerE13CudaStreamPtrb"><code class="docutils literal notranslate"><span class="pre">BufferManager()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManagerD0Ev"><code class="docutils literal notranslate"><span class="pre">~BufferManager()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">gpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager3gpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">gpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeNSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">allocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8allocateE10MemoryTypeN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">allocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyBufferE10MemoryTypeN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">emptyBuffer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager11emptyTensorE10MemoryTypeN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">emptyTensor()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager6setMemER7IBuffer7int32_t"><code class="docutils literal notranslate"><span class="pre">setMem()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager7setZeroER7IBuffer"><code class="docutils literal notranslate"><span class="pre">setZero()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer10MemoryType"><code class="docutils literal notranslate"><span class="pre">copy()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv10MemoryType"><code class="docutils literal notranslate"><span class="pre">copy()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyEPKvR7IBuffer"><code class="docutils literal notranslate"><span class="pre">copy()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferPv"><code class="docutils literal notranslate"><span class="pre">copy()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager4copyERK7IBufferR7IBuffer"><code class="docutils literal notranslate"><span class="pre">copy()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7IBuffer10MemoryType"><code class="docutils literal notranslate"><span class="pre">copyFrom()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager8copyFromERK7ITensor10MemoryType"><code class="docutils literal notranslate"><span class="pre">copyFrom()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10IBufferPtrRKNSt6vectorI1TEE10MemoryType"><code class="docutils literal notranslate"><span class="pre">copyFrom()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrP1TN8nvinfer14DimsE10MemoryType"><code class="docutils literal notranslate"><span class="pre">copyFrom()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime13BufferManager8copyFromE10ITensorPtrRKNSt6vectorI1TEEN8nvinfer14DimsE10MemoryType"><code class="docutils literal notranslate"><span class="pre">copyFrom()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager9getStreamEv"><code class="docutils literal notranslate"><span class="pre">getStream()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager18memoryPoolReservedEv"><code class="docutils literal notranslate"><span class="pre">memoryPoolReserved()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolUsedEv"><code class="docutils literal notranslate"><span class="pre">memoryPoolUsed()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13BufferManager14memoryPoolFreeEv"><code class="docutils literal notranslate"><span class="pre">memoryPoolFree()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager16memoryPoolTrimToENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">memoryPoolTrimTo()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">gpuSync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7gpuSyncEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">gpuSync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">cpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager3cpuEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">cpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">pinned()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager6pinnedEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">pinned()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">pinnedPool()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10pinnedPoolEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">pinnedPool()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7managedENSt6size_tEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">managed()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7managedEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">managed()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7ipcNvlsENSt3setIiEEN8nvinfer14DimsEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">ipcNvls()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager10kBYTE_TYPEE"><code class="docutils literal notranslate"><span class="pre">kBYTE_TYPE</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager7mStreamE"><code class="docutils literal notranslate"><span class="pre">mStream</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager5mPoolE"><code class="docutils literal notranslate"><span class="pre">mPool</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13BufferManager9mTrimPoolE"><code class="docutils literal notranslate"><span class="pre">mTrimPool</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#memorycounters-h">memoryCounters.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCountersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryCounters</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8DiffTypeE"><code class="docutils literal notranslate"><span class="pre">DiffType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters14MemoryCountersEv"><code class="docutils literal notranslate"><span class="pre">MemoryCounters()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getGpuEv"><code class="docutils literal notranslate"><span class="pre">getGpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getCpuEv"><code class="docutils literal notranslate"><span class="pre">getCpu()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters9getPinnedEv"><code class="docutils literal notranslate"><span class="pre">getPinned()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters6getUVMEv"><code class="docutils literal notranslate"><span class="pre">getUVM()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedPoolEv"><code class="docutils literal notranslate"><span class="pre">getPinnedPool()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getGpuDiffEv"><code class="docutils literal notranslate"><span class="pre">getGpuDiff()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getCpuDiffEv"><code class="docutils literal notranslate"><span class="pre">getCpuDiff()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters13getPinnedDiffEv"><code class="docutils literal notranslate"><span class="pre">getPinnedDiff()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters10getUVMDiffEv"><code class="docutils literal notranslate"><span class="pre">getUVMDiff()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters17getPinnedPoolDiffEv"><code class="docutils literal notranslate"><span class="pre">getPinnedPoolDiff()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters8allocateEv10SizeType32"><code class="docutils literal notranslate"><span class="pre">allocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8allocateE10MemoryType10SizeType32"><code class="docutils literal notranslate"><span class="pre">allocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime14MemoryCounters10deallocateEv10SizeType32"><code class="docutils literal notranslate"><span class="pre">deallocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters10deallocateE10MemoryType10SizeType32"><code class="docutils literal notranslate"><span class="pre">deallocate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14MemoryCounters8toStringEv"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11getInstanceEv"><code class="docutils literal notranslate"><span class="pre">getInstance()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE10SizeType32i"><code class="docutils literal notranslate"><span class="pre">bytesToString()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters13bytesToStringE8DiffTypei"><code class="docutils literal notranslate"><span class="pre">bytesToString()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mGpuE"><code class="docutils literal notranslate"><span class="pre">mGpu</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mCpuE"><code class="docutils literal notranslate"><span class="pre">mCpu</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters7mPinnedE"><code class="docutils literal notranslate"><span class="pre">mPinned</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters4mUVME"><code class="docutils literal notranslate"><span class="pre">mUVM</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedPoolE"><code class="docutils literal notranslate"><span class="pre">mPinnedPool</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mGpuDiffE"><code class="docutils literal notranslate"><span class="pre">mGpuDiff</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mCpuDiffE"><code class="docutils literal notranslate"><span class="pre">mCpuDiff</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters11mPinnedDiffE"><code class="docutils literal notranslate"><span class="pre">mPinnedDiff</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters8mUVMDiffE"><code class="docutils literal notranslate"><span class="pre">mUVMDiff</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14MemoryCounters15mPinnedPoolDiffE"><code class="docutils literal notranslate"><span class="pre">mPinnedPoolDiff</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#runtimedefaults-h">runtimeDefaults.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaultsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::RuntimeDefaults</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsENSt8optionalINSt6vectorI10SizeType32EEEENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">RuntimeDefaults()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15RuntimeDefaultsEv"><code class="docutils literal notranslate"><span class="pre">RuntimeDefaults()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults21maxAttentionWindowVecE"><code class="docutils literal notranslate"><span class="pre">maxAttentionWindowVec</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15RuntimeDefaults15sinkTokenLengthE"><code class="docutils literal notranslate"><span class="pre">sinkTokenLength</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ipcutils-h">ipcUtils.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20lamportInitializeAllEPvPvPv6size_t"><code class="docutils literal notranslate"><span class="pre">lamportInitializeAll()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13canAccessPeerERK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">canAccessPeer()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::AllReduceBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers16AllReduceBuffersE10SizeType3210SizeType3210SizeType3210SizeType32RK13BufferManagerRK11WorldConfigKb"><code class="docutils literal notranslate"><span class="pre">AllReduceBuffers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers18mAllReduceCommPtrsE"><code class="docutils literal notranslate"><span class="pre">mAllReduceCommPtrs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16AllReduceBuffers17mIpcMemoryHandlesE"><code class="docutils literal notranslate"><span class="pre">mIpcMemoryHandles</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::IpcMemory</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9BufferPtrE"><code class="docutils literal notranslate"><span class="pre">BufferPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfigb"><code class="docutils literal notranslate"><span class="pre">IpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryD0Ev"><code class="docutils literal notranslate"><span class="pre">~IpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERK9IpcMemory"><code class="docutils literal notranslate"><span class="pre">IpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERK9IpcMemory"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9IpcMemoryERR9IpcMemory"><code class="docutils literal notranslate"><span class="pre">IpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemoryaSERR9IpcMemory"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9IpcMemory11getCommPtrsEv"><code class="docutils literal notranslate"><span class="pre">getCommPtrs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory10FLAGS_SIZEE"><code class="docutils literal notranslate"><span class="pre">FLAGS_SIZE</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory17allocateIpcMemoryENSt6size_tERK13BufferManagerRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">allocateIpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory16destroyIpcMemoryEv"><code class="docutils literal notranslate"><span class="pre">destroyIpcMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory7mTpRankE"><code class="docutils literal notranslate"><span class="pre">mTpRank</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory9mCommPtrsE"><code class="docutils literal notranslate"><span class="pre">mCommPtrs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory7mBufferE"><code class="docutils literal notranslate"><span class="pre">mBuffer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9IpcMemory8mOpenIpcE"><code class="docutils literal notranslate"><span class="pre">mOpenIpc</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#tllmlogger-h">tllmLogger.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10TllmLoggerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TllmLogger</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger3logE8SeverityPKN8nvinfer19AsciiCharE"><code class="docutils literal notranslate"><span class="pre">log()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger8getLevelEv"><code class="docutils literal notranslate"><span class="pre">getLevel()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10TllmLogger8setLevelE8Severity"><code class="docutils literal notranslate"><span class="pre">setLevel()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gptdecoder-h">gptDecoder.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm6layersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::layers</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20getDefaultBatchSlotsEN7runtime10SizeType32E"><code class="docutils literal notranslate"><span class="pre">getDefaultBatchSlots()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime10GptDecoderE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::GptDecoder</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">CudaStreamPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder10GptDecoderERKN8executor12DecodingModeE6size_t6size_t6size_t6size_t6size_tRK13CudaStreamPtrNSt10shared_ptrIK25SpeculativeDecodingModuleEE"><code class="docutils literal notranslate"><span class="pre">GptDecoder()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"><code class="docutils literal notranslate"><span class="pre">setup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"><code class="docutils literal notranslate"><span class="pre">forwardAsync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"><code class="docutils literal notranslate"><span class="pre">forwardSync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder17getSamplingConfigEv"><code class="docutils literal notranslate"><span class="pre">getSamplingConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"><code class="docutils literal notranslate"><span class="pre">disableLookahead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder8mManagerE"><code class="docutils literal notranslate"><span class="pre">mManager</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder19mDynamicDecodeLayerE"><code class="docutils literal notranslate"><span class="pre">mDynamicDecodeLayer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder23mDecodingLayerWorkspaceE"><code class="docutils literal notranslate"><span class="pre">mDecodingLayerWorkspace</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder15mSamplingConfigE"><code class="docutils literal notranslate"><span class="pre">mSamplingConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13mMaxBatchSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxBatchSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder10mVocabSizeE"><code class="docutils literal notranslate"><span class="pre">mVocabSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder16mVocabSizePaddedE"><code class="docutils literal notranslate"><span class="pre">mVocabSizePadded</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10GptDecoder13mDecodingModeE"><code class="docutils literal notranslate"><span class="pre">mDecodingMode</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::IGptDecoder</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder14TensorConstPtrE"><code class="docutils literal notranslate"><span class="pre">TensorConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoderD0Ev"><code class="docutils literal notranslate"><span class="pre">~IGptDecoder()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder5setupERK14SamplingConfig6size_tRK14TensorConstPtrRKNSt8optionalI14DecodingOutputEERKNSt8optionalIKNSt6vectorIN13decoder_batch7RequestEEEEE"><code class="docutils literal notranslate"><span class="pre">setup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder12forwardAsyncER14DecodingOutputRK13DecodingInput"><code class="docutils literal notranslate"><span class="pre">forwardAsync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder11forwardSyncER14DecodingOutputRK13DecodingInput"><code class="docutils literal notranslate"><span class="pre">forwardSync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder17getSamplingConfigEv"><code class="docutils literal notranslate"><span class="pre">getSamplingConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder16disableLookaheadERKNSt8optionalI14SamplingConfigEE10SizeType3214TensorConstPtr"><code class="docutils literal notranslate"><span class="pre">disableLookahead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11IGptDecoder6createERKN8executor12DecodingModeEN8nvinfer18DataTypeE6size_t6size_t6size_t6size_t6size_tRKN13BufferManager13CudaStreamPtrERKNSt10shared_ptrIK25SpeculativeDecodingModuleEE"><code class="docutils literal notranslate"><span class="pre">create()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#cudaevent-h">cudaEvent.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEventE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::CudaEvent</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7pointerE"><code class="docutils literal notranslate"><span class="pre">pointer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventEj"><code class="docutils literal notranslate"><span class="pre">CudaEvent()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent9CudaEventE7pointerb"><code class="docutils literal notranslate"><span class="pre">CudaEvent()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent3getEv"><code class="docutils literal notranslate"><span class="pre">get()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent11synchronizeEv"><code class="docutils literal notranslate"><span class="pre">synchronize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent12element_typeE"><code class="docutils literal notranslate"><span class="pre">element_type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent8EventPtrE"><code class="docutils literal notranslate"><span class="pre">EventPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent6mEventE"><code class="docutils literal notranslate"><span class="pre">mEvent</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7DeleterE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::CudaEvent::Deleter</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEb"><code class="docutils literal notranslate"><span class="pre">Deleter()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter7DeleterEv"><code class="docutils literal notranslate"><span class="pre">Deleter()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9CudaEvent7DeleterclE7pointer"><code class="docutils literal notranslate"><span class="pre">operator()()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9CudaEvent7Deleter10mOwnsEventE"><code class="docutils literal notranslate"><span class="pre">mOwnsEvent</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#modelconfig-h">modelConfig.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ModelConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariantE"><code class="docutils literal notranslate"><span class="pre">ModelVariant</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGptE"><code class="docutils literal notranslate"><span class="pre">kGpt</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant8kChatGlmE"><code class="docutils literal notranslate"><span class="pre">kChatGlm</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant4kGlmE"><code class="docutils literal notranslate"><span class="pre">kGlm</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant6kMambaE"><code class="docutils literal notranslate"><span class="pre">kMamba</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant15kRecurrentGemmaE"><code class="docutils literal notranslate"><span class="pre">kRecurrentGemma</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12ModelVariant7kEncDecE"><code class="docutils literal notranslate"><span class="pre">kEncDec</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerTypeE"><code class="docutils literal notranslate"><span class="pre">LayerType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kATTENTIONE"><code class="docutils literal notranslate"><span class="pre">kATTENTION</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType10kRECURRENTE"><code class="docutils literal notranslate"><span class="pre">kRECURRENT</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType7kLINEARE"><code class="docutils literal notranslate"><span class="pre">kLINEAR</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9LayerType5kNOOPE"><code class="docutils literal notranslate"><span class="pre">kNOOP</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheTypeE"><code class="docutils literal notranslate"><span class="pre">KVCacheType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType11kCONTINUOUSE"><code class="docutils literal notranslate"><span class="pre">kCONTINUOUS</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType6kPAGEDE"><code class="docutils literal notranslate"><span class="pre">kPAGED</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11KVCacheType9kDISABLEDE"><code class="docutils literal notranslate"><span class="pre">kDISABLED</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsTypeE"><code class="docutils literal notranslate"><span class="pre">ManageWeightsType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType9kDisabledE"><code class="docutils literal notranslate"><span class="pre">kDisabled</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17ManageWeightsType8kEnabledE"><code class="docutils literal notranslate"><span class="pre">kEnabled</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11ModelConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">ModelConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getVocabSizeEv"><code class="docutils literal notranslate"><span class="pre">getVocabSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getVocabSizePaddedE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getVocabSizePadded()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16countLocalLayersE9LayerType10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">countLocalLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getFirstLocalLayerE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">getFirstLocalLayer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20countLowerRankLayersE9LayerType10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">countLowerRankLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getNbLayersE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">getNbLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getNbAttentionLayersE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">getNbAttentionLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getNbRnnLayersE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">getNbRnnLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig10getNbHeadsEv"><code class="docutils literal notranslate"><span class="pre">getNbHeads()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getNbKvHeadsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getNbKvHeads()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setNbKvHeadsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setNbKvHeads()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setNbCrossKvHeadsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setNbCrossKvHeads()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getHiddenSizeEv"><code class="docutils literal notranslate"><span class="pre">getHiddenSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getEncoderHiddenSizeEv"><code class="docutils literal notranslate"><span class="pre">getEncoderHiddenSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setEncoderHiddenSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setEncoderHiddenSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getSizePerHeadEv"><code class="docutils literal notranslate"><span class="pre">getSizePerHead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setSizePerHeadE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setSizePerHead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEv"><code class="docutils literal notranslate"><span class="pre">useGptAttentionPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEv"><code class="docutils literal notranslate"><span class="pre">useGemmAllReducePlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21useGptAttentionPluginEb"><code class="docutils literal notranslate"><span class="pre">useGptAttentionPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22useGemmAllReducePluginEb"><code class="docutils literal notranslate"><span class="pre">useGemmAllReducePlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEv"><code class="docutils literal notranslate"><span class="pre">useMambaConv1dPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20useMambaConv1dPluginEb"><code class="docutils literal notranslate"><span class="pre">useMambaConv1dPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14usePackedInputEv"><code class="docutils literal notranslate"><span class="pre">usePackedInput()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14usePackedInputEb"><code class="docutils literal notranslate"><span class="pre">usePackedInput()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13usePagedStateEv"><code class="docutils literal notranslate"><span class="pre">usePagedState()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13usePagedStateEb"><code class="docutils literal notranslate"><span class="pre">usePagedState()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getTokensPerBlockEv"><code class="docutils literal notranslate"><span class="pre">getTokensPerBlock()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setTokensPerBlockE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setTokensPerBlock()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getQuantModeEv"><code class="docutils literal notranslate"><span class="pre">getQuantMode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setQuantModeEN6common9QuantModeE"><code class="docutils literal notranslate"><span class="pre">setQuantMode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig24supportsInflightBatchingEv"><code class="docutils literal notranslate"><span class="pre">supportsInflightBatching()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBatchSizeEv"><code class="docutils literal notranslate"><span class="pre">getMaxBatchSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBatchSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxBatchSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxBeamWidthEv"><code class="docutils literal notranslate"><span class="pre">getMaxBeamWidth()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxBeamWidthE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxBeamWidth()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxInputLenEv"><code class="docutils literal notranslate"><span class="pre">getMaxInputLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxInputLenE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxInputLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17getMaxSequenceLenEv"><code class="docutils literal notranslate"><span class="pre">getMaxSequenceLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17setMaxSequenceLenE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxSequenceLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getMaxNumTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxNumTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setMaxNumTokensENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setMaxNumTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMaxEncoderLenEv"><code class="docutils literal notranslate"><span class="pre">getMaxEncoderLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMaxEncoderLenE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxEncoderLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15usePromptTuningEv"><code class="docutils literal notranslate"><span class="pre">usePromptTuning()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig8useMropeEv"><code class="docutils literal notranslate"><span class="pre">useMrope()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11setUseMropeEb"><code class="docutils literal notranslate"><span class="pre">setUseMrope()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig24getMaxPositionEmbeddingsEv"><code class="docutils literal notranslate"><span class="pre">getMaxPositionEmbeddings()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24setMaxPositionEmbeddingsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxPositionEmbeddings()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getRotaryEmbeddingDimEv"><code class="docutils literal notranslate"><span class="pre">getRotaryEmbeddingDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setRotaryEmbeddingDimE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setRotaryEmbeddingDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig30getMaxPromptEmbeddingTableSizeEv"><code class="docutils literal notranslate"><span class="pre">getMaxPromptEmbeddingTableSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig30setMaxPromptEmbeddingTableSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxPromptEmbeddingTableSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEv"><code class="docutils literal notranslate"><span class="pre">computeContextLogits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20computeContextLogitsEb"><code class="docutils literal notranslate"><span class="pre">computeContextLogits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEv"><code class="docutils literal notranslate"><span class="pre">computeGenerationLogits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23computeGenerationLogitsEb"><code class="docutils literal notranslate"><span class="pre">computeGenerationLogits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getModelVariantEv"><code class="docutils literal notranslate"><span class="pre">getModelVariant()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setModelVariantE12ModelVariant"><code class="docutils literal notranslate"><span class="pre">setModelVariant()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig25getMaxDecodingDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getMaxDecodingTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setContextFMHAEb"><code class="docutils literal notranslate"><span class="pre">setContextFMHA()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getContextFMHAEv"><code class="docutils literal notranslate"><span class="pre">getContextFMHA()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19setPagedContextFMHAEb"><code class="docutils literal notranslate"><span class="pre">setPagedContextFMHA()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19getPagedContextFMHAEv"><code class="docutils literal notranslate"><span class="pre">getPagedContextFMHA()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18setPpReduceScatterEb"><code class="docutils literal notranslate"><span class="pre">setPpReduceScatter()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getPpReduceScatterEv"><code class="docutils literal notranslate"><span class="pre">getPpReduceScatter()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13useLoraPluginEv"><code class="docutils literal notranslate"><span class="pre">useLoraPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13useLoraPluginEb"><code class="docutils literal notranslate"><span class="pre">useLoraPlugin()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLoraModulesEv"><code class="docutils literal notranslate"><span class="pre">getLoraModules()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLoraModulesERKNSt6vectorI10LoraModuleEE"><code class="docutils literal notranslate"><span class="pre">setLoraModules()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16getMlpHiddenSizeEv"><code class="docutils literal notranslate"><span class="pre">getMlpHiddenSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16setMlpHiddenSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMlpHiddenSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig16isKVCacheEnabledEv"><code class="docutils literal notranslate"><span class="pre">isKVCacheEnabled()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14isPagedKVCacheEv"><code class="docutils literal notranslate"><span class="pre">isPagedKVCache()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19isContinuousKVCacheEv"><code class="docutils literal notranslate"><span class="pre">isContinuousKVCache()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getKVCacheTypeEv"><code class="docutils literal notranslate"><span class="pre">getKVCacheType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setKVCacheTypeE11KVCacheType"><code class="docutils literal notranslate"><span class="pre">setKVCacheType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useCrossAttentionEv"><code class="docutils literal notranslate"><span class="pre">useCrossAttention()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseCrossAttentionEb"><code class="docutils literal notranslate"><span class="pre">setUseCrossAttention()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20usePositionEmbeddingEv"><code class="docutils literal notranslate"><span class="pre">usePositionEmbedding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23setUsePositionEmbeddingEb"><code class="docutils literal notranslate"><span class="pre">setUsePositionEmbedding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21useTokenTypeEmbeddingEv"><code class="docutils literal notranslate"><span class="pre">useTokenTypeEmbedding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24setUseTokenTypeEmbeddingEb"><code class="docutils literal notranslate"><span class="pre">setUseTokenTypeEmbedding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getMaxLoraRankEv"><code class="docutils literal notranslate"><span class="pre">getMaxLoraRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setMaxLoraRankE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxLoraRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26setSpeculativeDecodingModeE23SpeculativeDecodingMode"><code class="docutils literal notranslate"><span class="pre">setSpeculativeDecodingMode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig28hasSpeculativeDecodingModuleEv"><code class="docutils literal notranslate"><span class="pre">hasSpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig28getSpeculativeDecodingModuleEv"><code class="docutils literal notranslate"><span class="pre">getSpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"><code class="docutils literal notranslate"><span class="pre">getSpeculativeDecodingModulePtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig31getSpeculativeDecodingModulePtrEv"><code class="docutils literal notranslate"><span class="pre">getSpeculativeDecodingModulePtr()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28setSpeculativeDecodingModuleERKNSt10shared_ptrI25SpeculativeDecodingModuleEE"><code class="docutils literal notranslate"><span class="pre">setSpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig30resetSpeculativeDecodingModuleEv"><code class="docutils literal notranslate"><span class="pre">resetSpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig31enableSeamlessLookaheadDecodingE10SizeType32"><code class="docutils literal notranslate"><span class="pre">enableSeamlessLookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig32disableSeamlessLookaheadDecodingEv"><code class="docutils literal notranslate"><span class="pre">disableSeamlessLookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getKvDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getKvDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18isTransformerBasedEv"><code class="docutils literal notranslate"><span class="pre">isTransformerBased()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12hasRnnConfigEv"><code class="docutils literal notranslate"><span class="pre">hasRnnConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getRnnConfigEv"><code class="docutils literal notranslate"><span class="pre">getRnnConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setRnnConfigERK9RnnConfig"><code class="docutils literal notranslate"><span class="pre">setRnnConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig10isRnnBasedEv"><code class="docutils literal notranslate"><span class="pre">isRnnBased()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig13getLayerTypesEv"><code class="docutils literal notranslate"><span class="pre">getLayerTypes()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13setLayerTypesERKNSt6vectorI9LayerTypeEE"><code class="docutils literal notranslate"><span class="pre">setLayerTypes()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig26getSpeculativeDecodingModeEv"><code class="docutils literal notranslate"><span class="pre">getSpeculativeDecodingMode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14setLogitsDtypeEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">setLogitsDtype()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig14getLogitsDtypeEv"><code class="docutils literal notranslate"><span class="pre">getLogitsDtype()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setGemmAllReduceDtypeEN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">setGemmAllReduceDtype()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getGemmAllReduceDtypeEv"><code class="docutils literal notranslate"><span class="pre">getGemmAllReduceDtype()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setUseShapeInferenceEb"><code class="docutils literal notranslate"><span class="pre">setUseShapeInference()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig17useShapeInferenceEv"><code class="docutils literal notranslate"><span class="pre">useShapeInference()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig20getManageWeightsTypeEv"><code class="docutils literal notranslate"><span class="pre">getManageWeightsType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20setManageWeightsTypeEK17ManageWeightsType"><code class="docutils literal notranslate"><span class="pre">setManageWeightsType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12getModelNameEv"><code class="docutils literal notranslate"><span class="pre">getModelName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12setModelNameERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">setModelName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig21getNumKvHeadsPerLayerEv"><code class="docutils literal notranslate"><span class="pre">getNumKvHeadsPerLayer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig31getNumKvHeadsPerLayerLocalRangeE10SizeType3210SizeType32b"><code class="docutils literal notranslate"><span class="pre">getNumKvHeadsPerLayerLocalRange()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21setNumKvHeadsPerLayerERKNSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setNumKvHeadsPerLayer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26setNumKvHeadsPerCrossLayerERKNSt6vectorI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setNumKvHeadsPerCrossLayer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18getSumLocalKvHeadsE10SizeType3210SizeType32b"><code class="docutils literal notranslate"><span class="pre">getSumLocalKvHeads()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig19skipCrossAttnBlocksEv"><code class="docutils literal notranslate"><span class="pre">skipCrossAttnBlocks()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22setSkipCrossAttnBlocksEb"><code class="docutils literal notranslate"><span class="pre">setSkipCrossAttnBlocks()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig15getNumLanguagesEv"><code class="docutils literal notranslate"><span class="pre">getNumLanguages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig18useLanguageAdapterEv"><code class="docutils literal notranslate"><span class="pre">useLanguageAdapter()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15setNumLanguagesENSt8optionalI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">setNumLanguages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig12isMultiModalEv"><code class="docutils literal notranslate"><span class="pre">isMultiModal()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11ModelConfig9isWhisperEv"><code class="docutils literal notranslate"><span class="pre">isWhisper()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21KVCacheTypeFromStringENSt6stringE"><code class="docutils literal notranslate"><span class="pre">KVCacheTypeFromString()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig25getOptProfilesSplitPointsEv"><code class="docutils literal notranslate"><span class="pre">getOptProfilesSplitPoints()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26kOPT_PROFILES_SPLIT_POINTSE"><code class="docutils literal notranslate"><span class="pre">kOPT_PROFILES_SPLIT_POINTS</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig29kDEFAULT_NUM_TOKENS_PER_BLOCKE"><code class="docutils literal notranslate"><span class="pre">kDEFAULT_NUM_TOKENS_PER_BLOCK</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mVocabSizeE"><code class="docutils literal notranslate"><span class="pre">mVocabSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mNbLayersE"><code class="docutils literal notranslate"><span class="pre">mNbLayers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mNbAttentionLayersE"><code class="docutils literal notranslate"><span class="pre">mNbAttentionLayers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mNbRnnLayersE"><code class="docutils literal notranslate"><span class="pre">mNbRnnLayers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig8mNbHeadsE"><code class="docutils literal notranslate"><span class="pre">mNbHeads</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mHiddenSizeE"><code class="docutils literal notranslate"><span class="pre">mHiddenSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mSizePerHeadE"><code class="docutils literal notranslate"><span class="pre">mSizePerHead</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mDataTypeE"><code class="docutils literal notranslate"><span class="pre">mDataType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseGptAttentionPluginE"><code class="docutils literal notranslate"><span class="pre">mUseGptAttentionPlugin</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig23mUseGemmAllReducePluginE"><code class="docutils literal notranslate"><span class="pre">mUseGemmAllReducePlugin</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19mGemmAllReduceDtypeE"><code class="docutils literal notranslate"><span class="pre">mGemmAllReduceDtype</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUseMambaConv1dPluginE"><code class="docutils literal notranslate"><span class="pre">mUseMambaConv1dPlugin</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mInputPackedE"><code class="docutils literal notranslate"><span class="pre">mInputPacked</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mPagedStateE"><code class="docutils literal notranslate"><span class="pre">mPagedState</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15mTokensPerBlockE"><code class="docutils literal notranslate"><span class="pre">mTokensPerBlock</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mQuantModeE"><code class="docutils literal notranslate"><span class="pre">mQuantMode</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBatchSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxBatchSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxBeamWidthE"><code class="docutils literal notranslate"><span class="pre">mMaxBeamWidth</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxInputLenE"><code class="docutils literal notranslate"><span class="pre">mMaxInputLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig15mMaxSequenceLenE"><code class="docutils literal notranslate"><span class="pre">mMaxSequenceLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mMaxNumTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxNumTokens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mComputeContextLogitsE"><code class="docutils literal notranslate"><span class="pre">mComputeContextLogits</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24mComputeGenerationLogitsE"><code class="docutils literal notranslate"><span class="pre">mComputeGenerationLogits</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mModelVariantE"><code class="docutils literal notranslate"><span class="pre">mModelVariant</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28mMaxPromptEmbeddingTableSizeE"><code class="docutils literal notranslate"><span class="pre">mMaxPromptEmbeddingTableSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9mUseMropeE"><code class="docutils literal notranslate"><span class="pre">mUseMrope</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mMaxPositionEmbeddingsE"><code class="docutils literal notranslate"><span class="pre">mMaxPositionEmbeddings</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig19mRotaryEmbeddingDimE"><code class="docutils literal notranslate"><span class="pre">mRotaryEmbeddingDim</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mContextFMHAE"><code class="docutils literal notranslate"><span class="pre">mContextFMHA</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig17mPagedContextFMHAE"><code class="docutils literal notranslate"><span class="pre">mPagedContextFMHA</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig16mPpReduceScatterE"><code class="docutils literal notranslate"><span class="pre">mPpReduceScatter</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mUseLoraPluginE"><code class="docutils literal notranslate"><span class="pre">mUseLoraPlugin</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLoraModulesE"><code class="docutils literal notranslate"><span class="pre">mLoraModules</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMlpHiddenSizeE"><code class="docutils literal notranslate"><span class="pre">mMlpHiddenSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mMaxLoraRankE"><code class="docutils literal notranslate"><span class="pre">mMaxLoraRank</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mRnnConfigE"><code class="docutils literal notranslate"><span class="pre">mRnnConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mKVCacheTypeE"><code class="docutils literal notranslate"><span class="pre">mKVCacheType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig14mMaxEncoderLenE"><code class="docutils literal notranslate"><span class="pre">mMaxEncoderLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mEncoderHiddenSizeE"><code class="docutils literal notranslate"><span class="pre">mEncoderHiddenSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseCrossAttentionE"><code class="docutils literal notranslate"><span class="pre">mUseCrossAttention</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig21mUsePositionEmbeddingE"><code class="docutils literal notranslate"><span class="pre">mUsePositionEmbedding</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig22mUseTokenTypeEmbeddingE"><code class="docutils literal notranslate"><span class="pre">mUseTokenTypeEmbedding</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig11mLayerTypesE"><code class="docutils literal notranslate"><span class="pre">mLayerTypes</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig26mSpeculativeDecodingModuleE"><code class="docutils literal notranslate"><span class="pre">mSpeculativeDecodingModule</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig24mSpeculativeDecodingModeE"><code class="docutils literal notranslate"><span class="pre">mSpeculativeDecodingMode</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig12mLogitsDtypeE"><code class="docutils literal notranslate"><span class="pre">mLogitsDtype</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mUseShapeInferenceE"><code class="docutils literal notranslate"><span class="pre">mUseShapeInference</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig18mManageWeightsTypeE"><code class="docutils literal notranslate"><span class="pre">mManageWeightsType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig10mModelNameE"><code class="docutils literal notranslate"><span class="pre">mModelName</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig28mNumKvHeadsPerAttentionLayerE"><code class="docutils literal notranslate"><span class="pre">mNumKvHeadsPerAttentionLayer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig33mNumKvHeadsPerCrossAttentionLayerE"><code class="docutils literal notranslate"><span class="pre">mNumKvHeadsPerCrossAttentionLayer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig20mSkipCrossAttnBlocksE"><code class="docutils literal notranslate"><span class="pre">mSkipCrossAttnBlocks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig13mNumLanguagesE"><code class="docutils literal notranslate"><span class="pre">mNumLanguages</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ModelConfig::RnnConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig9stateSizeE"><code class="docutils literal notranslate"><span class="pre">stateSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig10convKernelE"><code class="docutils literal notranslate"><span class="pre">convKernel</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig13rnnHiddenSizeE"><code class="docutils literal notranslate"><span class="pre">rnnHiddenSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig11rnnHeadSizeE"><code class="docutils literal notranslate"><span class="pre">rnnHeadSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ModelConfig9RnnConfig14rnnConvDimSizeE"><code class="docutils literal notranslate"><span class="pre">rnnConvDimSize</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loracachepagemanagerconfig-h">loraCachePageManagerConfig.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK26LoraCachePageManagerConfig"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9to_stringERK26LoraCachePageManagerConfig"><code class="docutils literal notranslate"><span class="pre">to_string()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraCachePageManagerConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig26LoraCachePageManagerConfigEN7runtime10MemoryTypeEN8nvinfer18DataTypeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">LoraCachePageManagerConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getMemoryTypeEv"><code class="docutils literal notranslate"><span class="pre">getMemoryType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setMemoryTypeERKN7runtime10MemoryTypeE"><code class="docutils literal notranslate"><span class="pre">setMemoryType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11setDataTypeERKN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">setDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig16getTotalNumPagesEv"><code class="docutils literal notranslate"><span class="pre">getTotalNumPages()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setTotalNumPageERK10SizeType32"><code class="docutils literal notranslate"><span class="pre">setTotalNumPage()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig19getMaxPagesPerBlockEv"><code class="docutils literal notranslate"><span class="pre">getMaxPagesPerBlock()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig19setMaxPagesPerBlockERK10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxPagesPerBlock()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig15getSlotsPerPageEv"><code class="docutils literal notranslate"><span class="pre">getSlotsPerPage()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15setSlotsPerPageERK10SizeType32"><code class="docutils literal notranslate"><span class="pre">setSlotsPerPage()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig12getPageWidthEv"><code class="docutils literal notranslate"><span class="pre">getPageWidth()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig12setPageWidthERK10SizeType32"><code class="docutils literal notranslate"><span class="pre">setPageWidth()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig13getInitToZeroEv"><code class="docutils literal notranslate"><span class="pre">getInitToZero()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13setInitToZeroEb"><code class="docutils literal notranslate"><span class="pre">setInitToZero()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26LoraCachePageManagerConfig17getNumCopyStreamsEv"><code class="docutils literal notranslate"><span class="pre">getNumCopyStreams()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17setNumCopyStreamsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setNumCopyStreams()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mMemoryTypeE"><code class="docutils literal notranslate"><span class="pre">mMemoryType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig9mDataTypeE"><code class="docutils literal notranslate"><span class="pre">mDataType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig14mTotalNumPagesE"><code class="docutils literal notranslate"><span class="pre">mTotalNumPages</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig17mMaxPagesPerBlockE"><code class="docutils literal notranslate"><span class="pre">mMaxPagesPerBlock</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig13mSlotsPerPageE"><code class="docutils literal notranslate"><span class="pre">mSlotsPerPage</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig10mPageWidthE"><code class="docutils literal notranslate"><span class="pre">mPageWidth</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig15mNumCopyStreamsE"><code class="docutils literal notranslate"><span class="pre">mNumCopyStreams</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26LoraCachePageManagerConfig11mInitToZeroE"><code class="docutils literal notranslate"><span class="pre">mInitToZero</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#worldconfig-h">worldConfig.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::WorldConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig11WorldConfigE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType32RKNSt8optionalINSt6vectorI10SizeType32EEEEb"><code class="docutils literal notranslate"><span class="pre">WorldConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getSizeEv"><code class="docutils literal notranslate"><span class="pre">getSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig20getTensorParallelismEv"><code class="docutils literal notranslate"><span class="pre">getTensorParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig16isTensorParallelEv"><code class="docutils literal notranslate"><span class="pre">isTensorParallel()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getPipelineParallelismEv"><code class="docutils literal notranslate"><span class="pre">getPipelineParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig18isPipelineParallelEv"><code class="docutils literal notranslate"><span class="pre">isPipelineParallel()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getContextParallelismEv"><code class="docutils literal notranslate"><span class="pre">getContextParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig17isContextParallelEv"><code class="docutils literal notranslate"><span class="pre">isContextParallel()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig7getRankEv"><code class="docutils literal notranslate"><span class="pre">getRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig14getGpusPerNodeEv"><code class="docutils literal notranslate"><span class="pre">getGpusPerNode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig15getGpusPerGroupEv"><code class="docutils literal notranslate"><span class="pre">getGpusPerGroup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig9getDeviceEv"><code class="docutils literal notranslate"><span class="pre">getDevice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getDeviceOfE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getDeviceOf()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getPipelineParallelRankEv"><code class="docutils literal notranslate"><span class="pre">getPipelineParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig21getTensorParallelRankEv"><code class="docutils literal notranslate"><span class="pre">getTensorParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getContextParallelRankEv"><code class="docutils literal notranslate"><span class="pre">getContextParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig12getLocalRankEv"><code class="docutils literal notranslate"><span class="pre">getLocalRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getNodeRankEv"><code class="docutils literal notranslate"><span class="pre">getNodeRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig13getNodeRankOfE10SizeType32"><code class="docutils literal notranslate"><span class="pre">getNodeRankOf()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig27isFirstPipelineParallelRankEv"><code class="docutils literal notranslate"><span class="pre">isFirstPipelineParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isLastPipelineParallelRankEv"><code class="docutils literal notranslate"><span class="pre">isLastPipelineParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig25isFirstTensorParallelRankEv"><code class="docutils literal notranslate"><span class="pre">isFirstTensorParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig26isFirstContextParallelRankEv"><code class="docutils literal notranslate"><span class="pre">isFirstContextParallelRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig11getLastRankEv"><code class="docutils literal notranslate"><span class="pre">getLastRank()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig17enableAttentionDPEv"><code class="docutils literal notranslate"><span class="pre">enableAttentionDP()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig24getPipelineParallelGroupEv"><code class="docutils literal notranslate"><span class="pre">getPipelineParallelGroup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig22getTensorParallelGroupEv"><code class="docutils literal notranslate"><span class="pre">getTensorParallelGroup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig23getContextParallelGroupEv"><code class="docutils literal notranslate"><span class="pre">getContextParallelGroup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11WorldConfig14validMpiConfigEv"><code class="docutils literal notranslate"><span class="pre">validMpiConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig3mpiE10SizeType32NSt8optionalI10SizeType32EENSt8optionalI10SizeType32EENSt8optionalI10SizeType32EERKNSt8optionalINSt6vectorI10SizeType32EEEEb"><code class="docutils literal notranslate"><span class="pre">mpi()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19kDefaultGpusPerNodeE"><code class="docutils literal notranslate"><span class="pre">kDefaultGpusPerNode</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig18mTensorParallelismE"><code class="docutils literal notranslate"><span class="pre">mTensorParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig20mPipelineParallelismE"><code class="docutils literal notranslate"><span class="pre">mPipelineParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig19mContextParallelismE"><code class="docutils literal notranslate"><span class="pre">mContextParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig5mRankE"><code class="docutils literal notranslate"><span class="pre">mRank</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig12mGpusPerNodeE"><code class="docutils literal notranslate"><span class="pre">mGpusPerNode</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig18mEnableAttentionDPE"><code class="docutils literal notranslate"><span class="pre">mEnableAttentionDP</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11WorldConfig10mDeviceIdsE"><code class="docutils literal notranslate"><span class="pre">mDeviceIds</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#eaglemodule-h">eagleModule.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModuleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::EagleModule</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleE10SizeType3210SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">EagleModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule11EagleModuleEv"><code class="docutils literal notranslate"><span class="pre">EagleModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule22getDefaultEagleChoicesEv"><code class="docutils literal notranslate"><span class="pre">getDefaultEagleChoices()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule23getNumTransformerLayersEv"><code class="docutils literal notranslate"><span class="pre">getNumTransformerLayers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11EagleModule26getMaxNonLeafNodesPerLayerEv"><code class="docutils literal notranslate"><span class="pre">getMaxNonLeafNodesPerLayer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule21mNumTransformersLayerE"><code class="docutils literal notranslate"><span class="pre">mNumTransformersLayer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule24mMaxNonLeafNodesPerLayerE"><code class="docutils literal notranslate"><span class="pre">mMaxNonLeafNodesPerLayer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11EagleModule20mDefaultEagleChoicesE"><code class="docutils literal notranslate"><span class="pre">mDefaultEagleChoices</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#decodinginput-h">decodingInput.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14TensorConstPtrE"><code class="docutils literal notranslate"><span class="pre">TensorConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13DecodingInputE10SizeType3210SizeType3210SizeType3210SizeType3214TensorConstPtr9TensorPtr14TensorConstPtr"><code class="docutils literal notranslate"><span class="pre">DecodingInput()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput4stepE"><code class="docutils literal notranslate"><span class="pre">step</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9maxLengthE"><code class="docutils literal notranslate"><span class="pre">maxLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput18maxAttentionWindowE"><code class="docutils literal notranslate"><span class="pre">maxAttentionWindow</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15sinkTokenLengthE"><code class="docutils literal notranslate"><span class="pre">sinkTokenLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9batchSizeE"><code class="docutils literal notranslate"><span class="pre">batchSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput10beamWidthsE"><code class="docutils literal notranslate"><span class="pre">beamWidths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15maxStopWordsLenE"><code class="docutils literal notranslate"><span class="pre">maxStopWordsLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14maxBadWordsLenE"><code class="docutils literal notranslate"><span class="pre">maxBadWordsLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput6logitsE"><code class="docutils literal notranslate"><span class="pre">logits</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput9logitsVecE"><code class="docutils literal notranslate"><span class="pre">logitsVec</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput6endIdsE"><code class="docutils literal notranslate"><span class="pre">endIds</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput10batchSlotsE"><code class="docutils literal notranslate"><span class="pre">batchSlots</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13finishReasonsE"><code class="docutils literal notranslate"><span class="pre">finishReasons</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput19sequenceLimitLengthE"><code class="docutils literal notranslate"><span class="pre">sequenceLimitLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13embeddingBiasE"><code class="docutils literal notranslate"><span class="pre">embeddingBias</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput7lengthsE"><code class="docutils literal notranslate"><span class="pre">lengths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13badWordsListsE"><code class="docutils literal notranslate"><span class="pre">badWordsLists</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsPtrsE"><code class="docutils literal notranslate"><span class="pre">badWordsPtrs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12badWordsLensE"><code class="docutils literal notranslate"><span class="pre">badWordsLens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput14stopWordsListsE"><code class="docutils literal notranslate"><span class="pre">stopWordsLists</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsPtrsE"><code class="docutils literal notranslate"><span class="pre">stopWordsPtrs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput13stopWordsLensE"><code class="docutils literal notranslate"><span class="pre">stopWordsLens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput17noRepeatNgramSizeE"><code class="docutils literal notranslate"><span class="pre">noRepeatNgramSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput16cacheIndirectionE"><code class="docutils literal notranslate"><span class="pre">cacheIndirection</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15generationStepsE"><code class="docutils literal notranslate"><span class="pre">generationSteps</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12medusaInputsE"><code class="docutils literal notranslate"><span class="pre">medusaInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25explicitDraftTokensInputsE"><code class="docutils literal notranslate"><span class="pre">explicitDraftTokensInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15lookaheadInputsE"><code class="docutils literal notranslate"><span class="pre">lookaheadInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25externalDraftTokensInputsE"><code class="docutils literal notranslate"><span class="pre">externalDraftTokensInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11eagleInputsE"><code class="docutils literal notranslate"><span class="pre">eagleInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput::EagleInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs11EagleInputsE14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr14TensorConstPtr"><code class="docutils literal notranslate"><span class="pre">EagleInputs()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15nextDraftTokensE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13nextDraftLensE"><code class="docutils literal notranslate"><span class="pre">nextDraftLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14nextDraftPathsE"><code class="docutils literal notranslate"><span class="pre">nextDraftPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15lastDraftTokensE"><code class="docutils literal notranslate"><span class="pre">lastDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs13lastDraftLensE"><code class="docutils literal notranslate"><span class="pre">lastDraftLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14lastDraftPathsE"><code class="docutils literal notranslate"><span class="pre">lastDraftPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs14acceptedTokensE"><code class="docutils literal notranslate"><span class="pre">acceptedTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs12acceptedLensE"><code class="docutils literal notranslate"><span class="pre">acceptedLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs15acceptedPathIdsE"><code class="docutils literal notranslate"><span class="pre">acceptedPathIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs24chunkedContextNextTokensE"><code class="docutils literal notranslate"><span class="pre">chunkedContextNextTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput11EagleInputs8seqSlotsE"><code class="docutils literal notranslate"><span class="pre">seqSlots</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput::ExplicitDraftTokensInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15nextDraftTokensE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextFlatTokensE"><code class="docutils literal notranslate"><span class="pre">nextFlatTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16nextDraftIndicesE"><code class="docutils literal notranslate"><span class="pre">nextDraftIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs14nextDraftProbsE"><code class="docutils literal notranslate"><span class="pre">nextDraftProbs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15lastDraftTokensE"><code class="docutils literal notranslate"><span class="pre">lastDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs16lastDraftIndicesE"><code class="docutils literal notranslate"><span class="pre">lastDraftIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs5masksE"><code class="docutils literal notranslate"><span class="pre">masks</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs17packedPositionIdsE"><code class="docutils literal notranslate"><span class="pre">packedPositionIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathLengthsE"><code class="docutils literal notranslate"><span class="pre">bestPathLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs15bestPathIndicesE"><code class="docutils literal notranslate"><span class="pre">bestPathIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21nextGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">nextGenerationLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs19lastPositionIdsBaseE"><code class="docutils literal notranslate"><span class="pre">lastPositionIdsBase</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs21lastGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">lastGenerationLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs18maxGenLengthDeviceE"><code class="docutils literal notranslate"><span class="pre">maxGenLengthDevice</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExplicitDraftTokensInputs8seqSlotsE"><code class="docutils literal notranslate"><span class="pre">seqSlots</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput::ExternalDraftTokensInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11draftLogitsE"><code class="docutils literal notranslate"><span class="pre">draftLogits</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs10draftProbsE"><code class="docutils literal notranslate"><span class="pre">draftProbs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs11targetProbsE"><code class="docutils literal notranslate"><span class="pre">targetProbs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14numDraftTokensE"><code class="docutils literal notranslate"><span class="pre">numDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18numDraftTokensHostE"><code class="docutils literal notranslate"><span class="pre">numDraftTokensHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs13draftTokenIdsE"><code class="docutils literal notranslate"><span class="pre">draftTokenIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs14useDraftLogitsE"><code class="docutils literal notranslate"><span class="pre">useDraftLogits</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs18useDraftLogitsHostE"><code class="docutils literal notranslate"><span class="pre">useDraftLogitsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs4stepE"><code class="docutils literal notranslate"><span class="pre">step</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs17constantThresholdE"><code class="docutils literal notranslate"><span class="pre">constantThreshold</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput25ExternalDraftTokensInputs28useRandomAcceptanceThresholdE"><code class="docutils literal notranslate"><span class="pre">useRandomAcceptanceThreshold</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput::LookaheadInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput15LookaheadInputs13tokensPerStepE"><code class="docutils literal notranslate"><span class="pre">tokensPerStep</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingInput::MedusaInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs11medusaPathsE"><code class="docutils literal notranslate"><span class="pre">medusaPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs13medusaTreeIdsE"><code class="docutils literal notranslate"><span class="pre">medusaTreeIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs12medusaLogitsE"><code class="docutils literal notranslate"><span class="pre">medusaLogits</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs22medusaCurTokensPerStepE"><code class="docutils literal notranslate"><span class="pre">medusaCurTokensPerStep</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13DecodingInput12MedusaInputs25medusaTargetTokensPerStepE"><code class="docutils literal notranslate"><span class="pre">medusaTargetTokensPerStep</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gptjsonconfig-h">gptJsonConfig.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::GptJsonConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig13GptJsonConfigENSt6stringENSt6stringENSt6stringE10SizeType3210SizeType3210SizeType3210SizeType3211ModelConfigNSt8optionalI15RuntimeDefaultsEE"><code class="docutils literal notranslate"><span class="pre">GptJsonConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getModelConfigEv"><code class="docutils literal notranslate"><span class="pre">getModelConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig21getModelConfigMutableEv"><code class="docutils literal notranslate"><span class="pre">getModelConfigMutable()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig7getNameEv"><code class="docutils literal notranslate"><span class="pre">getName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig10getVersionEv"><code class="docutils literal notranslate"><span class="pre">getVersion()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getPrecisionEv"><code class="docutils literal notranslate"><span class="pre">getPrecision()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig20getTensorParallelismEv"><code class="docutils literal notranslate"><span class="pre">getTensorParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig22getPipelineParallelismEv"><code class="docutils literal notranslate"><span class="pre">getPipelineParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig21getContextParallelismEv"><code class="docutils literal notranslate"><span class="pre">getContextParallelism()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14getGpusPerNodeEv"><code class="docutils literal notranslate"><span class="pre">getGpusPerNode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig12getWorldSizeEv"><code class="docutils literal notranslate"><span class="pre">getWorldSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig18getRuntimeDefaultsEv"><code class="docutils literal notranslate"><span class="pre">getRuntimeDefaults()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfigRKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">engineFilename()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime13GptJsonConfig14engineFilenameERK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">engineFilename()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt6stringE"><code class="docutils literal notranslate"><span class="pre">parse()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERNSt7istreamE"><code class="docutils literal notranslate"><span class="pre">parse()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5parseERKNSt10filesystem4pathE"><code class="docutils literal notranslate"><span class="pre">parse()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig5mNameE"><code class="docutils literal notranslate"><span class="pre">mName</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig8mVersionE"><code class="docutils literal notranslate"><span class="pre">mVersion</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig10mPrecisionE"><code class="docutils literal notranslate"><span class="pre">mPrecision</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig18mTensorParallelismE"><code class="docutils literal notranslate"><span class="pre">mTensorParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig20mPipelineParallelismE"><code class="docutils literal notranslate"><span class="pre">mPipelineParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig19mContextParallelismE"><code class="docutils literal notranslate"><span class="pre">mContextParallelism</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mGpusPerNodeE"><code class="docutils literal notranslate"><span class="pre">mGpusPerNode</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig12mModelConfigE"><code class="docutils literal notranslate"><span class="pre">mModelConfig</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13GptJsonConfig16mRuntimeDefaultsE"><code class="docutils literal notranslate"><span class="pre">mRuntimeDefaults</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ipcnvlsmemory-h">ipcNvlsMemory.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17MPI_group_barrierENSt3setIiEE"><code class="docutils literal notranslate"><span class="pre">MPI_group_barrier()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16ipcNvlsSupportedEv"><code class="docutils literal notranslate"><span class="pre">ipcNvlsSupported()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15ipcNvlsAllocateE6size_tNSt3setIiEE"><code class="docutils literal notranslate"><span class="pre">ipcNvlsAllocate()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11ipcNvlsFreeEP13IpcNvlsHandle"><code class="docutils literal notranslate"><span class="pre">ipcNvlsFree()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime20DeviceAllocationNvlsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DeviceAllocationNvls</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls20DeviceAllocationNvlsEv"><code class="docutils literal notranslate"><span class="pre">DeviceAllocationNvls()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvlsD0Ev"><code class="docutils literal notranslate"><span class="pre">~DeviceAllocationNvls()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls5resetE6size_tNSt3setIiEE"><code class="docutils literal notranslate"><span class="pre">reset()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls19getMulticastPointerEv"><code class="docutils literal notranslate"><span class="pre">getMulticastPointer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls17getUnicastPointerEv"><code class="docutils literal notranslate"><span class="pre">getUnicastPointer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls21getIpcUnicastPointersEv"><code class="docutils literal notranslate"><span class="pre">getIpcUnicastPointers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime20DeviceAllocationNvls11getCapacityEv"><code class="docutils literal notranslate"><span class="pre">getCapacity()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls4freeEv"><code class="docutils literal notranslate"><span class="pre">free()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls9_capacityE"><code class="docutils literal notranslate"><span class="pre">_capacity</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime20DeviceAllocationNvls7_handleE"><code class="docutils literal notranslate"><span class="pre">_handle</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::IpcNvlsHandle</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6uc_ptrE"><code class="docutils literal notranslate"><span class="pre">uc_ptr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle6mc_ptrE"><code class="docutils literal notranslate"><span class="pre">mc_ptr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle11ipc_uc_ptrsE"><code class="docutils literal notranslate"><span class="pre">ipc_uc_ptrs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5uc_vaE"><code class="docutils literal notranslate"><span class="pre">uc_va</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle5mc_vaE"><code class="docutils literal notranslate"><span class="pre">mc_va</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle10ipc_uc_vasE"><code class="docutils literal notranslate"><span class="pre">ipc_uc_vas</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9uc_handleE"><code class="docutils literal notranslate"><span class="pre">uc_handle</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle9mc_handleE"><code class="docutils literal notranslate"><span class="pre">mc_handle</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13IpcNvlsHandle14ipc_uc_handlesE"><code class="docutils literal notranslate"><span class="pre">ipc_uc_handles</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#samplingconfig-h">samplingConfig.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#c.SET_FROM_OPTIONAL"><code class="docutils literal notranslate"><span class="pre">SET_FROM_OPTIONAL</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfigE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::SamplingConfig</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigE10SizeType32"><code class="docutils literal notranslate"><span class="pre">SamplingConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKNSt6vectorI14SamplingConfigEE"><code class="docutils literal notranslate"><span class="pre">SamplingConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14SamplingConfigERKN8executor14SamplingConfigERKNSt8optionalIN8executor25ExternalDraftTokensConfigEEE"><code class="docutils literal notranslate"><span class="pre">SamplingConfig()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig8validateEv"><code class="docutils literal notranslate"><span class="pre">validate()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig16useDefaultValuesEbRK6OptVecI1TE1T"><code class="docutils literal notranslate"><span class="pre">useDefaultValues()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfigeqERK14SamplingConfig"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfig17getNumReturnBeamsEv"><code class="docutils literal notranslate"><span class="pre">getNumReturnBeams()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14SamplingConfig15getMaxBeamWidthEv"><code class="docutils literal notranslate"><span class="pre">getMaxBeamWidth()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9beamWidthE"><code class="docutils literal notranslate"><span class="pre">beamWidth</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig18numReturnSequencesE"><code class="docutils literal notranslate"><span class="pre">numReturnSequences</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig11temperatureE"><code class="docutils literal notranslate"><span class="pre">temperature</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig19originalTemperatureE"><code class="docutils literal notranslate"><span class="pre">originalTemperature</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9minLengthE"><code class="docutils literal notranslate"><span class="pre">minLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17repetitionPenaltyE"><code class="docutils literal notranslate"><span class="pre">repetitionPenalty</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig15presencePenaltyE"><code class="docutils literal notranslate"><span class="pre">presencePenalty</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig16frequencyPenaltyE"><code class="docutils literal notranslate"><span class="pre">frequencyPenalty</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17noRepeatNgramSizeE"><code class="docutils literal notranslate"><span class="pre">noRepeatNgramSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14outputLogProbsE"><code class="docutils literal notranslate"><span class="pre">outputLogProbs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig11cumLogProbsE"><code class="docutils literal notranslate"><span class="pre">cumLogProbs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topKE"><code class="docutils literal notranslate"><span class="pre">topK</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4topPE"><code class="docutils literal notranslate"><span class="pre">topP</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig10randomSeedE"><code class="docutils literal notranslate"><span class="pre">randomSeed</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9topPDecayE"><code class="docutils literal notranslate"><span class="pre">topPDecay</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig7topPMinE"><code class="docutils literal notranslate"><span class="pre">topPMin</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig12topPResetIdsE"><code class="docutils literal notranslate"><span class="pre">topPResetIds</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig4minPE"><code class="docutils literal notranslate"><span class="pre">minP</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig23beamSearchDiversityRateE"><code class="docutils literal notranslate"><span class="pre">beamSearchDiversityRate</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig13lengthPenaltyE"><code class="docutils literal notranslate"><span class="pre">lengthPenalty</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig13earlyStoppingE"><code class="docutils literal notranslate"><span class="pre">earlyStopping</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig14beamWidthArrayE"><code class="docutils literal notranslate"><span class="pre">beamWidthArray</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig24draftAcceptanceThresholdE"><code class="docutils literal notranslate"><span class="pre">draftAcceptanceThreshold</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig15topKMedusaHeadsE"><code class="docutils literal notranslate"><span class="pre">topKMedusaHeads</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig17normalizeLogProbsE"><code class="docutils literal notranslate"><span class="pre">normalizeLogProbs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14SamplingConfig9FloatTypeE"><code class="docutils literal notranslate"><span class="pre">FloatType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig6OptVecE"><code class="docutils literal notranslate"><span class="pre">OptVec</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig11validateVecEbNSt6stringERK6OptVecI1TE1TNSt8optionalI1TEE"><code class="docutils literal notranslate"><span class="pre">validateVec()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime14SamplingConfig10fuseValuesE6OptVecI1TERKNSt6vectorI14SamplingConfigEENSt8functionIF6OptVecI1TE6size_tEEE1T"><code class="docutils literal notranslate"><span class="pre">fuseValues()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gptdecoderbatched-h">gptDecoderBatched.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm13batch_managerE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::batch_manager</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatchedE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::GptDecoderBatched</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">CudaStreamPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13LlmRequestPtrE"><code class="docutils literal notranslate"><span class="pre">LlmRequestPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13RequestVectorE"><code class="docutils literal notranslate"><span class="pre">RequestVector</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14SharedConstPtrE"><code class="docutils literal notranslate"><span class="pre">SharedConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched17GptDecoderBatchedE13CudaStreamPtrRK23SpeculativeDecodingModeN8nvinfer18DataTypeE"><code class="docutils literal notranslate"><span class="pre">GptDecoderBatched()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">setup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"><code class="docutils literal notranslate"><span class="pre">disableLookahead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">forwardAsync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">forward()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"><code class="docutils literal notranslate"><span class="pre">finalize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"><code class="docutils literal notranslate"><span class="pre">getDecoderState()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched15getDecoderStateEv"><code class="docutils literal notranslate"><span class="pre">getDecoderState()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getDecoderStreamEv"><code class="docutils literal notranslate"><span class="pre">getDecoderStream()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched20getUnderlyingDecoderEv"><code class="docutils literal notranslate"><span class="pre">getUnderlyingDecoder()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime17GptDecoderBatched16getBufferManagerEv"><code class="docutils literal notranslate"><span class="pre">getBufferManager()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13GptDecoderPtrE"><code class="docutils literal notranslate"><span class="pre">GptDecoderPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched28setExplicitDraftTokensInputsERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">setExplicitDraftTokensInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14setEagleInputsERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">setEagleInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched15forwardDispatchERN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">forwardDispatch()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14prepareForwardE10SizeType32RN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">prepareForward()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mRuntimeStreamE"><code class="docutils literal notranslate"><span class="pre">mRuntimeStream</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mDecoderStreamE"><code class="docutils literal notranslate"><span class="pre">mDecoderStream</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched14mBufferManagerE"><code class="docutils literal notranslate"><span class="pre">mBufferManager</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched8mDecoderE"><code class="docutils literal notranslate"><span class="pre">mDecoder</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime17GptDecoderBatched13mDecoderStateE"><code class="docutils literal notranslate"><span class="pre">mDecoderState</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#lookaheadbuffers-h">lookaheadBuffers.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LookaheadDecodingBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers24LookaheadDecodingBuffersE10SizeType3210SizeType32RK13BufferManager"><code class="docutils literal notranslate"><span class="pre">LookaheadDecodingBuffers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers17generationLengthsE"><code class="docutils literal notranslate"><span class="pre">generationLengths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers15positionOffsetsE"><code class="docutils literal notranslate"><span class="pre">positionOffsets</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11packedMasksE"><code class="docutils literal notranslate"><span class="pre">packedMasks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime24LookaheadDecodingBuffers11positionIdsE"><code class="docutils literal notranslate"><span class="pre">positionIds</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LookaheadRuntimeBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers9TensorMapE"><code class="docutils literal notranslate"><span class="pre">TensorMap</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23LookaheadRuntimeBuffersE10SizeType3210SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfigRKN8executor14DecodingConfigERK11TllmRuntime"><code class="docutils literal notranslate"><span class="pre">LookaheadRuntimeBuffers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers13setFromInputsE10SizeType3210SizeType32RK7ITensorRK7ITensorRK24LookaheadDecodingBuffersRK11TllmRuntimeRK11ModelConfigRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">setFromInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers7reshapeE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23LookaheadRuntimeBuffers18insertInputTensorsER9TensorMapR9TensorMapRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">insertInputTensors()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23enableLookaheadDecodingE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">enableLookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers24disableLookaheadDecodingEv"><code class="docutils literal notranslate"><span class="pre">disableLookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers12cumSumLengthE"><code class="docutils literal notranslate"><span class="pre">cumSumLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17packedMasksDeviceE"><code class="docutils literal notranslate"><span class="pre">packedMasksDevice</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23generationLengthsDeviceE"><code class="docutils literal notranslate"><span class="pre">generationLengthsDevice</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21positionOffsetsDeviceE"><code class="docutils literal notranslate"><span class="pre">positionOffsetsDevice</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers17positionIdsDeviceE"><code class="docutils literal notranslate"><span class="pre">positionIdsDevice</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers14packedMaskHostE"><code class="docutils literal notranslate"><span class="pre">packedMaskHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers21generationLengthsHostE"><code class="docutils literal notranslate"><span class="pre">generationLengthsHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionOffsetsHostE"><code class="docutils literal notranslate"><span class="pre">positionOffsetsHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15positionIdsHostE"><code class="docutils literal notranslate"><span class="pre">positionIdsHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18packedMaskHostCopyE"><code class="docutils literal notranslate"><span class="pre">packedMaskHostCopy</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers25generationLengthsHostCopyE"><code class="docutils literal notranslate"><span class="pre">generationLengthsHostCopy</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers23positionOffsetsHostCopyE"><code class="docutils literal notranslate"><span class="pre">positionOffsetsHostCopy</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers19positionIdsHostCopyE"><code class="docutils literal notranslate"><span class="pre">positionIdsHostCopy</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers15useSpecDecodingE"><code class="docutils literal notranslate"><span class="pre">useSpecDecoding</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23LookaheadRuntimeBuffers18batchSlotsHostCopyE"><code class="docutils literal notranslate"><span class="pre">batchSlotsHostCopy</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#loramodule-h">loraModule.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK10LoraModule"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::LoraModule</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleTypeE"><code class="docutils literal notranslate"><span class="pre">ModuleType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType8kINVALIDE"><code class="docutils literal notranslate"><span class="pre">kINVALID</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kATTN_QKVE"><code class="docutils literal notranslate"><span class="pre">kATTN_QKV</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_QE"><code class="docutils literal notranslate"><span class="pre">kATTN_Q</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_KE"><code class="docutils literal notranslate"><span class="pre">kATTN_K</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType7kATTN_VE"><code class="docutils literal notranslate"><span class="pre">kATTN_V</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kATTN_DENSEE"><code class="docutils literal notranslate"><span class="pre">kATTN_DENSE</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_H_TO_4HE"><code class="docutils literal notranslate"><span class="pre">kMLP_H_TO_4H</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_4H_TO_HE"><code class="docutils literal notranslate"><span class="pre">kMLP_4H_TO_H</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMLP_GATEE"><code class="docutils literal notranslate"><span class="pre">kMLP_GATE</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType15kCROSS_ATTN_QKVE"><code class="docutils literal notranslate"><span class="pre">kCROSS_ATTN_QKV</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_QE"><code class="docutils literal notranslate"><span class="pre">kCROSS_ATTN_Q</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_KE"><code class="docutils literal notranslate"><span class="pre">kCROSS_ATTN_K</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType13kCROSS_ATTN_VE"><code class="docutils literal notranslate"><span class="pre">kCROSS_ATTN_V</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType17kCROSS_ATTN_DENSEE"><code class="docutils literal notranslate"><span class="pre">kCROSS_ATTN_DENSE</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_H_TO_4HE"><code class="docutils literal notranslate"><span class="pre">kMOE_H_TO_4H</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMOE_4H_TO_HE"><code class="docutils literal notranslate"><span class="pre">kMOE_4H_TO_H</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType9kMOE_GATEE"><code class="docutils literal notranslate"><span class="pre">kMOE_GATE</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMOE_ROUTERE"><code class="docutils literal notranslate"><span class="pre">kMOE_ROUTER</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType11kMLP_ROUTERE"><code class="docutils literal notranslate"><span class="pre">kMLP_ROUTER</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10ModuleType12kMLP_GATE_UPE"><code class="docutils literal notranslate"><span class="pre">kMLP_GATE_UP</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10ModuleType10SizeType3210SizeType32bb10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">LoraModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleEv"><code class="docutils literal notranslate"><span class="pre">LoraModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule10LoraModuleERK10LoraModule"><code class="docutils literal notranslate"><span class="pre">LoraModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModuleaSERK10LoraModule"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule18flattenedInOutSizeE10SizeType32b"><code class="docutils literal notranslate"><span class="pre">flattenedInOutSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule6inSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">inSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule7outSizeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">outSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11localInSizeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">localInSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule12localOutSizeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">localOutSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule15localScalesSizeE10SizeType32b"><code class="docutils literal notranslate"><span class="pre">localScalesSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule10localInDimE10SizeType32"><code class="docutils literal notranslate"><span class="pre">localInDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11localOutDimE10SizeType32"><code class="docutils literal notranslate"><span class="pre">localOutDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule18localInAdapterSizeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">localInAdapterSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule19localOutAdapterSizeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">localOutAdapterSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule14localInOutSizeE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">localInOutSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule14localTotalSizeE10SizeType3210SizeType32b"><code class="docutils literal notranslate"><span class="pre">localTotalSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule5valueEv"><code class="docutils literal notranslate"><span class="pre">value()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule4nameEv"><code class="docutils literal notranslate"><span class="pre">name()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule5inDimEv"><code class="docutils literal notranslate"><span class="pre">inDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule6outDimEv"><code class="docutils literal notranslate"><span class="pre">outDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule10inDimFirstEv"><code class="docutils literal notranslate"><span class="pre">inDimFirst()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule11outDimFirstEv"><code class="docutils literal notranslate"><span class="pre">outDimFirst()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule12inTpSplitDimEv"><code class="docutils literal notranslate"><span class="pre">inTpSplitDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10LoraModule13outTpSplitDimEv"><code class="docutils literal notranslate"><span class="pre">outTpSplitDim()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule17createLoraModulesERKNSt6vectorINSt6stringEEE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">createLoraModules()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleTypeERKNSt11string_viewE"><code class="docutils literal notranslate"><span class="pre">toModuleType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10ModuleType"><code class="docutils literal notranslate"><span class="pre">toModuleName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12toModuleNameE10SizeType32"><code class="docutils literal notranslate"><span class="pre">toModuleName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule5mTypeE"><code class="docutils literal notranslate"><span class="pre">mType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule6mInDimE"><code class="docutils literal notranslate"><span class="pre">mInDim</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule7mOutDimE"><code class="docutils literal notranslate"><span class="pre">mOutDim</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule11mInDimFirstE"><code class="docutils literal notranslate"><span class="pre">mInDimFirst</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule12mOutDimFirstE"><code class="docutils literal notranslate"><span class="pre">mOutDimFirst</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule13mInTpSplitDimE"><code class="docutils literal notranslate"><span class="pre">mInTpSplitDim</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10LoraModule14mOutTpSplitDimE"><code class="docutils literal notranslate"><span class="pre">mOutTpSplitDim</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#prompttuningparams-h">promptTuningParams.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime25GenericPromptTuningParamsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::GenericPromptTuningParams</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams25GenericPromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"><code class="docutils literal notranslate"><span class="pre">GenericPromptTuningParams()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams14embeddingTableE"><code class="docutils literal notranslate"><span class="pre">embeddingTable</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams5tasksE"><code class="docutils literal notranslate"><span class="pre">tasks</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams9vocabSizeE"><code class="docutils literal notranslate"><span class="pre">vocabSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25GenericPromptTuningParams19promptTuningEnabledE"><code class="docutils literal notranslate"><span class="pre">promptTuningEnabled</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParamsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::PromptTuningParams</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams18PromptTuningParamsE9TensorPtr9TensorPtr9TensorPtr"><code class="docutils literal notranslate"><span class="pre">PromptTuningParams()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18PromptTuningParams15fillTasksTensorE9TensorPtr10SizeType3210SizeType32RKNSt6vectorI10SizeType32EERKNSt6vectorI10SizeType32EERK13BufferManagerb"><code class="docutils literal notranslate"><span class="pre">fillTasksTensor()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#speculativedecodingmode-h">speculativeDecodingMode.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingModeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::SpeculativeDecodingMode</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode14UnderlyingTypeE"><code class="docutils literal notranslate"><span class="pre">UnderlyingType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode6isNoneEv"><code class="docutils literal notranslate"><span class="pre">isNone()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isDraftTokensExternalEv"><code class="docutils literal notranslate"><span class="pre">isDraftTokensExternal()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode8isMedusaEv"><code class="docutils literal notranslate"><span class="pre">isMedusa()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19isLookaheadDecodingEv"><code class="docutils literal notranslate"><span class="pre">isLookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21isExplicitDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">isExplicitDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode7isEagleEv"><code class="docutils literal notranslate"><span class="pre">isEagle()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18updatesPositionIdsEv"><code class="docutils literal notranslate"><span class="pre">updatesPositionIds()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode21requiresAttentionMaskEv"><code class="docutils literal notranslate"><span class="pre">requiresAttentionMask()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19predictsDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">predictsDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode18needsKVCacheRewindEv"><code class="docutils literal notranslate"><span class="pre">needsKVCacheRewind()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode19variableDraftLengthEv"><code class="docutils literal notranslate"><span class="pre">variableDraftLength()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode14hasDraftLogitsEv"><code class="docutils literal notranslate"><span class="pre">hasDraftLogits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode20needsDecoderPrologueEv"><code class="docutils literal notranslate"><span class="pre">needsDecoderPrologue()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingModeeqERK23SpeculativeDecodingMode"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode23SpeculativeDecodingModeE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">SpeculativeDecodingMode()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode4NoneEv"><code class="docutils literal notranslate"><span class="pre">None()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19DraftTokensExternalEv"><code class="docutils literal notranslate"><span class="pre">DraftTokensExternal()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6MedusaEv"><code class="docutils literal notranslate"><span class="pre">Medusa()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode17LookaheadDecodingEv"><code class="docutils literal notranslate"><span class="pre">LookaheadDecoding()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode19ExplicitDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">ExplicitDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5EagleEv"><code class="docutils literal notranslate"><span class="pre">Eagle()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9anyBitSetE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">anyBitSet()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime23SpeculativeDecodingMode9allBitSetE14UnderlyingType"><code class="docutils literal notranslate"><span class="pre">allBitSet()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6mStateE"><code class="docutils literal notranslate"><span class="pre">mState</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode5kNoneE"><code class="docutils literal notranslate"><span class="pre">kNone</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kDraftTokensExternalE"><code class="docutils literal notranslate"><span class="pre">kDraftTokensExternal</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode7kMedusaE"><code class="docutils literal notranslate"><span class="pre">kMedusa</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode18kLookaheadDecodingE"><code class="docutils literal notranslate"><span class="pre">kLookaheadDecoding</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode20kExplicitDraftTokensE"><code class="docutils literal notranslate"><span class="pre">kExplicitDraftTokens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime23SpeculativeDecodingMode6kEagleE"><code class="docutils literal notranslate"><span class="pre">kEagle</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#common-h">common.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#c.FMT_DIM"><code class="docutils literal notranslate"><span class="pre">FMT_DIM</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10SizeType64E"><code class="docutils literal notranslate"><span class="pre">SizeType64</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TokenIdTypeE"><code class="docutils literal notranslate"><span class="pre">TokenIdType</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14LoraTaskIdTypeE"><code class="docutils literal notranslate"><span class="pre">LoraTaskIdType</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16TokenExtraIdTypeE"><code class="docutils literal notranslate"><span class="pre">TokenExtraIdType</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16VecTokenExtraIdsE"><code class="docutils literal notranslate"><span class="pre">VecTokenExtraIds</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime15VecUniqueTokensE"><code class="docutils literal notranslate"><span class="pre">VecUniqueTokens</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime12StringPtrMapE"><code class="docutils literal notranslate"><span class="pre">StringPtrMap</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11RequestTypeE"><code class="docutils literal notranslate"><span class="pre">RequestType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11RequestType8kCONTEXTE"><code class="docutils literal notranslate"><span class="pre">kCONTEXT</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11RequestType11kGENERATIONE"><code class="docutils literal notranslate"><span class="pre">kGENERATION</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11UniqueTokenE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::UniqueToken</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime11UniqueTokeneqERK11UniqueToken"><code class="docutils literal notranslate"><span class="pre">operator==()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11UniqueToken7tokenIdE"><code class="docutils literal notranslate"><span class="pre">tokenId</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11UniqueToken12tokenExtraIdE"><code class="docutils literal notranslate"><span class="pre">tokenExtraId</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#medusamodule-h">medusaModule.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModuleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MedusaModule</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule13MedusaChoicesE"><code class="docutils literal notranslate"><span class="pre">MedusaChoices</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">MedusaModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule12MedusaModuleEv"><code class="docutils literal notranslate"><span class="pre">MedusaModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime12MedusaModule16getMedusaChoicesEv"><code class="docutils literal notranslate"><span class="pre">getMedusaChoices()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12MedusaModule21mDefaultMedusaChoicesE"><code class="docutils literal notranslate"><span class="pre">mDefaultMedusaChoices</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#decodingoutput-h">decodingOutput.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingOutput</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14DecodingOutputE9TensorPtr9TensorPtr"><code class="docutils literal notranslate"><span class="pre">DecodingOutput()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput3idsE"><code class="docutils literal notranslate"><span class="pre">ids</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11gatheredIdsE"><code class="docutils literal notranslate"><span class="pre">gatheredIds</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14newTokensStepsE"><code class="docutils literal notranslate"><span class="pre">newTokensSteps</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9newTokensE"><code class="docutils literal notranslate"><span class="pre">newTokens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput12newTokensVecE"><code class="docutils literal notranslate"><span class="pre">newTokensVec</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput13finishReasonsE"><code class="docutils literal notranslate"><span class="pre">finishReasons</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11finishedSumE"><code class="docutils literal notranslate"><span class="pre">finishedSum</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput8logProbsE"><code class="docutils literal notranslate"><span class="pre">logProbs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput11cumLogProbsE"><code class="docutils literal notranslate"><span class="pre">cumLogProbs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput9parentIdsE"><code class="docutils literal notranslate"><span class="pre">parentIds</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput7lengthsE"><code class="docutils literal notranslate"><span class="pre">lengths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput16cacheIndirectionE"><code class="docutils literal notranslate"><span class="pre">cacheIndirection</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput13logProbsTiledE"><code class="docutils literal notranslate"><span class="pre">logProbsTiled</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14beamHypothesesE"><code class="docutils literal notranslate"><span class="pre">beamHypotheses</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26speculativeDecodingOutputsE"><code class="docutils literal notranslate"><span class="pre">speculativeDecodingOutputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26explicitDraftTokensBuffersE"><code class="docutils literal notranslate"><span class="pre">explicitDraftTokensBuffers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput16lookaheadOutputsE"><code class="docutils literal notranslate"><span class="pre">lookaheadOutputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput12eagleBuffersE"><code class="docutils literal notranslate"><span class="pre">eagleBuffers</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput17kNegativeInfinityE"><code class="docutils literal notranslate"><span class="pre">kNegativeInfinity</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypothesesE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingOutput::BeamHypotheses</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5emptyERK13BufferManager"><code class="docutils literal notranslate"><span class="pre">empty()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7reshapeE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses7releaseEv"><code class="docutils literal notranslate"><span class="pre">release()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses4initERK13BufferManager11TokenIdType"><code class="docutils literal notranslate"><span class="pre">init()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses5sliceE10SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses12outputIdsCBAE"><code class="docutils literal notranslate"><span class="pre">outputIdsCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11logProbsCBAE"><code class="docutils literal notranslate"><span class="pre">logProbsCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18sequenceLengthsCBAE"><code class="docutils literal notranslate"><span class="pre">sequenceLengthsCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses14cumLogProbsCBAE"><code class="docutils literal notranslate"><span class="pre">cumLogProbsCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses15normedScoresCBAE"><code class="docutils literal notranslate"><span class="pre">normedScoresCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses11numBeamsCBAE"><code class="docutils literal notranslate"><span class="pre">numBeamsCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses18minNormedScoresCBAE"><code class="docutils literal notranslate"><span class="pre">minNormedScoresCBA</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput14BeamHypotheses10batchDonesE"><code class="docutils literal notranslate"><span class="pre">batchDones</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DecodingOutput::SpeculativeDecodingOutputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs15nextDraftTokensE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18nextDraftTokensLenE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokensLen</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs18prevDraftTokensLenE"><code class="docutils literal notranslate"><span class="pre">prevDraftTokensLen</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs17acceptedTokensLenE"><code class="docutils literal notranslate"><span class="pre">acceptedTokensLen</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs21acceptedLengthsCumSumE"><code class="docutils literal notranslate"><span class="pre">acceptedLengthsCumSum</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DecodingOutput26SpeculativeDecodingOutputs12pathsOffsetsE"><code class="docutils literal notranslate"><span class="pre">pathsOffsets</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#cudastream-h">cudaStream.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStreamE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::CudaStream</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamEji"><code class="docutils literal notranslate"><span class="pre">CudaStream()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_tib"><code class="docutils literal notranslate"><span class="pre">CudaStream()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream10CudaStreamE12cudaStream_t"><code class="docutils literal notranslate"><span class="pre">CudaStream()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream9getDeviceEv"><code class="docutils literal notranslate"><span class="pre">getDevice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream3getEv"><code class="docutils literal notranslate"><span class="pre">get()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream11synchronizeEv"><code class="docutils literal notranslate"><span class="pre">synchronize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordEN9CudaEvent7pointerE"><code class="docutils literal notranslate"><span class="pre">record()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream6recordERK9CudaEvent"><code class="docutils literal notranslate"><span class="pre">record()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitEN9CudaEvent7pointerE"><code class="docutils literal notranslate"><span class="pre">wait()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream4waitERK9CudaEvent"><code class="docutils literal notranslate"><span class="pre">wait()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream9StreamPtrE"><code class="docutils literal notranslate"><span class="pre">StreamPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7mStreamE"><code class="docutils literal notranslate"><span class="pre">mStream</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7mDeviceE"><code class="docutils literal notranslate"><span class="pre">mDevice</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7DeleterE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::CudaStream::Deleter</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEb"><code class="docutils literal notranslate"><span class="pre">Deleter()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter7DeleterEv"><code class="docutils literal notranslate"><span class="pre">Deleter()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime10CudaStream7DeleterclE12cudaStream_t"><code class="docutils literal notranslate"><span class="pre">operator()()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10CudaStream7Deleter11mOwnsStreamE"><code class="docutils literal notranslate"><span class="pre">mOwnsStream</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#eaglebuffers-h">eagleBuffers.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::EagleBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13LlmRequestPtrE"><code class="docutils literal notranslate"><span class="pre">LlmRequestPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13RequestVectorE"><code class="docutils literal notranslate"><span class="pre">RequestVector</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7ITensorE"><code class="docutils literal notranslate"><span class="pre">ITensor</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9BufferPtrE"><code class="docutils literal notranslate"><span class="pre">BufferPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers9TensorMapE"><code class="docutils literal notranslate"><span class="pre">TensorMap</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers12EagleBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN8executor14DecodingConfigE"><code class="docutils literal notranslate"><span class="pre">EagleBuffers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime12EagleBuffers13setFromInputsERK13RequestVectorRK13RequestVectorRKN7runtime7ITensorERK7ITensorRKN12EagleBuffers6InputsERKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">setFromInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime12EagleBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">insertInputTensors()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers12engineInputsE"><code class="docutils literal notranslate"><span class="pre">engineInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13engineOutputsE"><code class="docutils literal notranslate"><span class="pre">engineOutputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime12EagleBuffers13setFromInputsEvRK13RequestVectorRK13RequestVector10SizeType32RK7ITensorRKN12EagleBuffers6InputsERKN7runtime11EagleModuleERKN7runtime13BufferManagerE"><code class="docutils literal notranslate"><span class="pre">setFromInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers26scanReduceTempStorageBytesE"><code class="docutils literal notranslate"><span class="pre">scanReduceTempStorageBytes</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers26mDefaultPosteriorThresholdE"><code class="docutils literal notranslate"><span class="pre">mDefaultPosteriorThreshold</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers17mDoGreedySamplingE"><code class="docutils literal notranslate"><span class="pre">mDoGreedySampling</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers21scanReduceTempStorageE"><code class="docutils literal notranslate"><span class="pre">scanReduceTempStorage</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers23cumSumGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">cumSumGenerationLengths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers19maxGenerationLengthE"><code class="docutils literal notranslate"><span class="pre">maxGenerationLength</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers28chunkedContextNextTokensHostE"><code class="docutils literal notranslate"><span class="pre">chunkedContextNextTokensHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers18greedySamplingHostE"><code class="docutils literal notranslate"><span class="pre">greedySamplingHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers18posteriorAlphaHostE"><code class="docutils literal notranslate"><span class="pre">posteriorAlphaHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers22posteriorThresholdHostE"><code class="docutils literal notranslate"><span class="pre">posteriorThresholdHost</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::EagleBuffers::EngineOutputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs15nextDraftTokensE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13nextDraftLensE"><code class="docutils literal notranslate"><span class="pre">nextDraftLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14nextDraftPathsE"><code class="docutils literal notranslate"><span class="pre">nextDraftPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs14acceptedTokensE"><code class="docutils literal notranslate"><span class="pre">acceptedTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs12acceptedLensE"><code class="docutils literal notranslate"><span class="pre">acceptedLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs13acceptedPathsE"><code class="docutils literal notranslate"><span class="pre">acceptedPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers13EngineOutputs24chunkedContextNextTokensE"><code class="docutils literal notranslate"><span class="pre">chunkedContextNextTokens</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6InputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::EagleBuffers::Inputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs6createE10SizeType32RK13BufferManagerRK11ModelConfigRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">create()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs12temperaturesE"><code class="docutils literal notranslate"><span class="pre">temperatures</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14posteriorAlphaE"><code class="docutils literal notranslate"><span class="pre">posteriorAlpha</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18posteriorThresholdE"><code class="docutils literal notranslate"><span class="pre">posteriorThreshold</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs16randomDataSampleE"><code class="docutils literal notranslate"><span class="pre">randomDataSample</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20randomDataValidationE"><code class="docutils literal notranslate"><span class="pre">randomDataValidation</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs11draftTokensE"><code class="docutils literal notranslate"><span class="pre">draftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs9draftLensE"><code class="docutils literal notranslate"><span class="pre">draftLens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10draftPathsE"><code class="docutils literal notranslate"><span class="pre">draftPaths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs14draftPathsHostE"><code class="docutils literal notranslate"><span class="pre">draftPathsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29specDecodingGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">specDecodingGenerationLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33specDecodingGenerationLengthsHostE"><code class="docutils literal notranslate"><span class="pre">specDecodingGenerationLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs23specDecodingPackedMasksE"><code class="docutils literal notranslate"><span class="pre">specDecodingPackedMasks</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27specDecodingPositionOffsetsE"><code class="docutils literal notranslate"><span class="pre">specDecodingPositionOffsets</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetCtxRequestTypesHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetCtxRequestTypesHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetCtxContextLengthsHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetCtxContextLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetCtxPastKeyValueLengthsHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetCtxPastKeyValueLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs27eagleNetGenRequestTypesHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetGenRequestTypesHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs29eagleNetGenContextLengthsHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetGenContextLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs34eagleNetGenPastKeyValueLengthsHostE"><code class="docutils literal notranslate"><span class="pre">eagleNetGenPastKeyValueLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18inputGenTokensHostE"><code class="docutils literal notranslate"><span class="pre">inputGenTokensHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs24chunkedContextNextTokensE"><code class="docutils literal notranslate"><span class="pre">chunkedContextNextTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15useSpecDecodingE"><code class="docutils literal notranslate"><span class="pre">useSpecDecoding</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs18useDynamicTreeHostE"><code class="docutils literal notranslate"><span class="pre">useDynamicTreeHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22dynamicTreeMaxTopKHostE"><code class="docutils literal notranslate"><span class="pre">dynamicTreeMaxTopKHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs10prevScoresE"><code class="docutils literal notranslate"><span class="pre">prevScores</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs20currentExpandIndicesE"><code class="docutils literal notranslate"><span class="pre">currentExpandIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs15allLayersScoresE"><code class="docutils literal notranslate"><span class="pre">allLayersScores</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs22allLayersDraftTokenIdsE"><code class="docutils literal notranslate"><span class="pre">allLayersDraftTokenIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime12EagleBuffers6Inputs33allLayersDraftTokenIdsPredecessorE"><code class="docutils literal notranslate"><span class="pre">allLayersDraftTokenIdsPredecessor</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#igptdecoderbatched-h">iGptDecoderBatched.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::IGptDecoderBatched</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13CudaStreamPtrE"><code class="docutils literal notranslate"><span class="pre">CudaStreamPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13LlmRequestPtrE"><code class="docutils literal notranslate"><span class="pre">LlmRequestPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched13RequestVectorE"><code class="docutils literal notranslate"><span class="pre">RequestVector</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched5setupERKN8executor12DecodingModeE10SizeType3210SizeType3210SizeType3210SizeType3210SizeType3210SizeType32N8nvinfer18DataTypeERK11ModelConfigRK11WorldConfig"><code class="docutils literal notranslate"><span class="pre">setup()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched16disableLookaheadERK13RequestVectorRK9TensorPtr"><code class="docutils literal notranslate"><span class="pre">disableLookahead()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched12forwardAsyncERN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">forwardAsync()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched7forwardERN13decoder_batch6OutputERKN13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">forward()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime18IGptDecoderBatched8finalizeERKN7decoder12DecoderStateE10SizeType32RK14SamplingConfigb"><code class="docutils literal notranslate"><span class="pre">finalize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatched18IGptDecoderBatchedEv"><code class="docutils literal notranslate"><span class="pre">IGptDecoderBatched()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime18IGptDecoderBatchedD0Ev"><code class="docutils literal notranslate"><span class="pre">~IGptDecoderBatched()</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5InputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder_batch::Input</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input14TensorConstPtrE"><code class="docutils literal notranslate"><span class="pre">TensorConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorINSt6vectorI14TensorConstPtrEEEE10SizeType32"><code class="docutils literal notranslate"><span class="pre">Input()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input5InputERKNSt6vectorI14TensorConstPtrEE"><code class="docutils literal notranslate"><span class="pre">Input()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input6logitsE"><code class="docutils literal notranslate"><span class="pre">logits</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15maxDecoderStepsE"><code class="docutils literal notranslate"><span class="pre">maxDecoderSteps</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input10batchSlotsE"><code class="docutils literal notranslate"><span class="pre">batchSlots</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input22batchSlotsRequestOrderE"><code class="docutils literal notranslate"><span class="pre">batchSlotsRequestOrder</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input16cacheIndirectionE"><code class="docutils literal notranslate"><span class="pre">cacheIndirection</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15generationStepsE"><code class="docutils literal notranslate"><span class="pre">generationSteps</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input20predictedDraftLogitsE"><code class="docutils literal notranslate"><span class="pre">predictedDraftLogits</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input25explicitDraftTokensInputsE"><code class="docutils literal notranslate"><span class="pre">explicitDraftTokensInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input29explicitDraftTokensLastInputsE"><code class="docutils literal notranslate"><span class="pre">explicitDraftTokensLastInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input11eagleInputsE"><code class="docutils literal notranslate"><span class="pre">eagleInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch5Input15eagleLastInputsE"><code class="docutils literal notranslate"><span class="pre">eagleLastInputs</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6OutputE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::decoder_batch::Output</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output6OutputEv"><code class="docutils literal notranslate"><span class="pre">Output()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime13decoder_batch6Output16cacheIndirectionE"><code class="docutils literal notranslate"><span class="pre">cacheIndirection</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#speculativedecodingmodule-h">speculativeDecodingModule.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::SpeculativeDecodingModule</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleE10SizeType3210SizeType3210SizeType32"><code class="docutils literal notranslate"><span class="pre">SpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleEv"><code class="docutils literal notranslate"><span class="pre">SpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleD0Ev"><code class="docutils literal notranslate"><span class="pre">~SpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule25SpeculativeDecodingModuleERK25SpeculativeDecodingModule"><code class="docutils literal notranslate"><span class="pre">SpeculativeDecodingModule()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModuleaSERK25SpeculativeDecodingModule"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule18getMaxDraftPathLenEv"><code class="docutils literal notranslate"><span class="pre">getMaxDraftPathLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule13getMaxPathLenEv"><code class="docutils literal notranslate"><span class="pre">getMaxPathLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule25getMaxDecodingDraftTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule20getMaxDecodingTokensEv"><code class="docutils literal notranslate"><span class="pre">getMaxDecodingTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule17getNumPackedMasksEv"><code class="docutils literal notranslate"><span class="pre">getNumPackedMasks()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime25SpeculativeDecodingModule14getMaxNumPathsEv"><code class="docutils literal notranslate"><span class="pre">getMaxNumPaths()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule17setMaxDraftTokensE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxDraftTokens()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18setMaxDraftPathLenE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxDraftPathLen()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule14setMaxNumPathsE10SizeType32"><code class="docutils literal notranslate"><span class="pre">setMaxNumPaths()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule21computeNumPackedMasksEv"><code class="docutils literal notranslate"><span class="pre">computeNumPackedMasks()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule16mMaxDraftPathLenE"><code class="docutils literal notranslate"><span class="pre">mMaxDraftPathLen</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule23mMaxDecodingDraftTokensE"><code class="docutils literal notranslate"><span class="pre">mMaxDecodingDraftTokens</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule12mMaxNumPathsE"><code class="docutils literal notranslate"><span class="pre">mMaxNumPaths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime25SpeculativeDecodingModule18mMaxNumPackedMasksE"><code class="docutils literal notranslate"><span class="pre">mMaxNumPackedMasks</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#explicitdrafttokensbuffers-h">explicitDraftTokensBuffers.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffersE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ExplicitDraftTokensBuffers</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers10SizeType32E"><code class="docutils literal notranslate"><span class="pre">SizeType32</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7ITensorE"><code class="docutils literal notranslate"><span class="pre">ITensor</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9BufferPtrE"><code class="docutils literal notranslate"><span class="pre">BufferPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorPtrE"><code class="docutils literal notranslate"><span class="pre">TensorPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers9TensorMapE"><code class="docutils literal notranslate"><span class="pre">TensorMap</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers26ExplicitDraftTokensBuffersE10SizeType3210SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">ExplicitDraftTokensBuffers()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers7reshapeE10SizeType3210SizeType32RKN7runtime11ModelConfigE"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsE10SizeType3210SizeType32RKN7runtime7ITensorERK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime11ModelConfigERKN7runtime11WorldConfigERKN7runtime13BufferManagerERKN7runtime10CudaStreamE"><code class="docutils literal notranslate"><span class="pre">setFromInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers18insertInputTensorsER9TensorMapR9TensorMapRKN7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">insertInputTensors()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12engineInputsE"><code class="docutils literal notranslate"><span class="pre">engineInputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13engineOutputsE"><code class="docutils literal notranslate"><span class="pre">engineOutputs</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers20scanTempStorageBytesE"><code class="docutils literal notranslate"><span class="pre">scanTempStorageBytes</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers15scanTempStorageE"><code class="docutils literal notranslate"><span class="pre">scanTempStorage</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers23cumSumGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">cumSumGenerationLengths</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13setFromInputsEv10SizeType3210SizeType3210SizeType32RK7ITensorRKN26ExplicitDraftTokensBuffers6InputsERK7ITensorRKN7runtime25ExplicitDraftTokensModuleERKN7runtime10CudaStreamE"><code class="docutils literal notranslate"><span class="pre">setFromInputs()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineInputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs18requestTypesDeviceE"><code class="docutils literal notranslate"><span class="pre">requestTypesDevice</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers12EngineInputs15positionOffsetsE"><code class="docutils literal notranslate"><span class="pre">positionOffsets</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ExplicitDraftTokensBuffers::EngineOutputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs21nextGenerationLengthsE"><code class="docutils literal notranslate"><span class="pre">nextGenerationLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs19nextPositionOffsetsE"><code class="docutils literal notranslate"><span class="pre">nextPositionOffsets</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs5masksE"><code class="docutils literal notranslate"><span class="pre">masks</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15nextDraftTokensE"><code class="docutils literal notranslate"><span class="pre">nextDraftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs16nextDraftIndicesE"><code class="docutils literal notranslate"><span class="pre">nextDraftIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextDraftProbsE"><code class="docutils literal notranslate"><span class="pre">nextDraftProbs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs14nextFlatTokensE"><code class="docutils literal notranslate"><span class="pre">nextFlatTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathLengthsE"><code class="docutils literal notranslate"><span class="pre">bestPathLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs15bestPathIndicesE"><code class="docutils literal notranslate"><span class="pre">bestPathIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs11maxGenTokenE"><code class="docutils literal notranslate"><span class="pre">maxGenToken</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs13totalGenTokenE"><code class="docutils literal notranslate"><span class="pre">totalGenToken</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers13EngineOutputs17packedPositionIdsE"><code class="docutils literal notranslate"><span class="pre">packedPositionIds</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6InputsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ExplicitDraftTokensBuffers::Inputs</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs6createE10SizeType32RKN7runtime13BufferManagerERKN7runtime11ModelConfigERKN7runtime11WorldConfigE"><code class="docutils literal notranslate"><span class="pre">create()</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12temperaturesE"><code class="docutils literal notranslate"><span class="pre">temperatures</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15positionIdsBaseE"><code class="docutils literal notranslate"><span class="pre">positionIdsBase</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs17generationLengthsE"><code class="docutils literal notranslate"><span class="pre">generationLengths</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16randomDataSampleE"><code class="docutils literal notranslate"><span class="pre">randomDataSample</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs20randomDataValidationE"><code class="docutils literal notranslate"><span class="pre">randomDataValidation</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11draftTokensE"><code class="docutils literal notranslate"><span class="pre">draftTokens</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs12draftIndicesE"><code class="docutils literal notranslate"><span class="pre">draftIndices</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs10draftProbsE"><code class="docutils literal notranslate"><span class="pre">draftProbs</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11packedMasksE"><code class="docutils literal notranslate"><span class="pre">packedMasks</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs11positionIdsE"><code class="docutils literal notranslate"><span class="pre">positionIds</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs16maxGenLengthHostE"><code class="docutils literal notranslate"><span class="pre">maxGenLengthHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs21generationLengthsHostE"><code class="docutils literal notranslate"><span class="pre">generationLengthsHost</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime26ExplicitDraftTokensBuffers6Inputs15useSpecDecodingE"><code class="docutils literal notranslate"><span class="pre">useSpecDecoding</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rawengine-h">rawEngine.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngineE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::RawEngine</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4TypeE"><code class="docutils literal notranslate"><span class="pre">Type</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type8FilePathE"><code class="docutils literal notranslate"><span class="pre">FilePath</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type15AddressWithSizeE"><code class="docutils literal notranslate"><span class="pre">AddressWithSize</span></code></a></li>
<li class="toc-h5 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine4Type10HostMemoryE"><code class="docutils literal notranslate"><span class="pre">HostMemory</span></code></a></li>
</ul>
</li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineENSt10filesystem4pathE"><code class="docutils literal notranslate"><span class="pre">RawEngine()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKvNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">RawEngine()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine9RawEngineEPKN8nvinfer111IHostMemoryE"><code class="docutils literal notranslate"><span class="pre">RawEngine()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getTypeEv"><code class="docutils literal notranslate"><span class="pre">getType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getPathEv"><code class="docutils literal notranslate"><span class="pre">getPath()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine10getPathOptEv"><code class="docutils literal notranslate"><span class="pre">getPathOpt()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine7setPathENSt10filesystem4pathE"><code class="docutils literal notranslate"><span class="pre">setPath()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine23getManagedWeightsMapOptEv"><code class="docutils literal notranslate"><span class="pre">getManagedWeightsMapOpt()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine20setManagedWeightsMapENSt3mapINSt6stringEN12tensorrt_llm8executor6TensorEEE"><code class="docutils literal notranslate"><span class="pre">setManagedWeightsMap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine10getAddressEv"><code class="docutils literal notranslate"><span class="pre">getAddress()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine7getSizeEv"><code class="docutils literal notranslate"><span class="pre">getSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime9RawEngine13getHostMemoryEv"><code class="docutils literal notranslate"><span class="pre">getHostMemory()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineAddrE"><code class="docutils literal notranslate"><span class="pre">mEngineAddr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEngineSizeE"><code class="docutils literal notranslate"><span class="pre">mEngineSize</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine5mTypeE"><code class="docutils literal notranslate"><span class="pre">mType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine11mEnginePathE"><code class="docutils literal notranslate"><span class="pre">mEnginePath</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine13mEngineBufferE"><code class="docutils literal notranslate"><span class="pre">mEngineBuffer</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime9RawEngine18mManagedWeightsMapE"><code class="docutils literal notranslate"><span class="pre">mManagedWeightsMap</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#itensor-h">iTensor.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv48nvinfer1"><code class="docutils literal notranslate"><span class="pre">nvinfer1</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERKN7ITensor5ShapeE"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7ITensor"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7ITensor14SharedConstPtrE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7ITensor9SharedPtrE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7ITensor9SharedPtrEEE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7ITensor14SharedConstPtrEEE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensorE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::ITensor</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9UniquePtrE"><code class="docutils literal notranslate"><span class="pre">UniquePtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9SharedPtrE"><code class="docutils literal notranslate"><span class="pre">SharedPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14UniqueConstPtrE"><code class="docutils literal notranslate"><span class="pre">UniqueConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor14SharedConstPtrE"><code class="docutils literal notranslate"><span class="pre">SharedConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5ShapeE"><code class="docutils literal notranslate"><span class="pre">Shape</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9DimType64E"><code class="docutils literal notranslate"><span class="pre">DimType64</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9TensorMapE"><code class="docutils literal notranslate"><span class="pre">TensorMap</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensorD0Ev"><code class="docutils literal notranslate"><span class="pre">~ITensor()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor8getShapeEv"><code class="docutils literal notranslate"><span class="pre">getShape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_10SizeType32ENK12tensorrt_llm7runtime7ITensor12getDimensionE9DimType64v"><code class="docutils literal notranslate"><span class="pre">getDimension()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7reshapeERK5Shape"><code class="docutils literal notranslate"><span class="pre">reshape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor6resizeENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">resize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorERK7ITensor"><code class="docutils literal notranslate"><span class="pre">ITensor()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensoraSERK7ITensor"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">squeeze()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeE10SizeType32"><code class="docutils literal notranslate"><span class="pre">unsqueeze()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERK5Shape"><code class="docutils literal notranslate"><span class="pre">shapeEquals()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7ITensor11shapeEqualsERKNSt16initializer_listI10SizeType32EE"><code class="docutils literal notranslate"><span class="pre">shapeEquals()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0ENK12tensorrt_llm7runtime7ITensor11shapeEqualsEbPK1T10SizeType32"><code class="docutils literal notranslate"><span class="pre">shapeEquals()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor6volumeERK5Shape"><code class="docutils literal notranslate"><span class="pre">volume()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor17volumeNonNegativeERK5Shape"><code class="docutils literal notranslate"><span class="pre">volumeNonNegative()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7stridesERK5Shape"><code class="docutils literal notranslate"><span class="pre">strides()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7squeezeERK5Shape10SizeType32"><code class="docutils literal notranslate"><span class="pre">squeeze()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9unsqueezeERK5Shape10SizeType32"><code class="docutils literal notranslate"><span class="pre">unsqueeze()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape9DimType64"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE9DimType64"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5ShapeNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRK5Shape"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor5sliceE9SharedPtrRKNSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRK5Shape"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor5sliceE14UniqueConstPtrRR9TConstPtrRKNSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRK5Shape"><code class="docutils literal notranslate"><span class="pre">at()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor2atE9SharedPtrRKNSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">at()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atE14UniqueConstPtrRR9TConstPtrRK5Shape"><code class="docutils literal notranslate"><span class="pre">at()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor2atEN7ITensor14UniqueConstPtrERR9TConstPtrRKNSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">at()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4viewEN7IBuffer9SharedPtrERK5Shape"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7ITensor4viewE14UniqueConstPtrRR9TConstPtrRK5Shape"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4viewE9SharedPtr"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8flattenNE9SharedPtrNSt7int64_tE"><code class="docutils literal notranslate"><span class="pre">flattenN()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5ShapeNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor4wrapEPvN8nvinfer18DataTypeERK5Shape"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5ShapeNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrP1TRK5Shape"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor4wrapE9UniquePtrRNSt6vectorI1TEERK5Shape"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor9makeShapeERKNSt16initializer_listI9DimType64EE"><code class="docutils literal notranslate"><span class="pre">makeShape()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8toStringERK5Shape"><code class="docutils literal notranslate"><span class="pre">toString()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor11shapeEqualsERK5ShapeRK5Shape"><code class="docutils literal notranslate"><span class="pre">shapeEquals()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7ITensor11shapeEqualsEbRK5ShapePK1T10SizeType32"><code class="docutils literal notranslate"><span class="pre">shapeEquals()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor7ITensorEv"><code class="docutils literal notranslate"><span class="pre">ITensor()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7ITensor8castSizeE6size_t"><code class="docutils literal notranslate"><span class="pre">castSize()</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ibuffer-h">iBuffer.h</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime18PointerElementTypeE"><code class="docutils literal notranslate"><span class="pre">PointerElementType</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryTypeE"><code class="docutils literal notranslate"><span class="pre">MemoryType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kGPUE"><code class="docutils literal notranslate"><span class="pre">kGPU</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kCPUE"><code class="docutils literal notranslate"><span class="pre">kCPU</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType7kPINNEDE"><code class="docutils literal notranslate"><span class="pre">kPINNED</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType4kUVME"><code class="docutils literal notranslate"><span class="pre">kUVM</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime10MemoryType11kPINNEDPOOLE"><code class="docutils literal notranslate"><span class="pre">kPINNEDPOOL</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERKNSt10shared_ptrI1TEE"><code class="docutils literal notranslate"><span class="pre">constPointerCast()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I00EN12tensorrt_llm7runtime16constPointerCastENSt10shared_ptrINSt14remove_const_tI1TEEEERRNSt10unique_ptrI1T1DEE"><code class="docutils literal notranslate"><span class="pre">constPointerCast()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEPK1TRK7IBuffer"><code class="docutils literal notranslate"><span class="pre">bufferCast()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime10bufferCastEP1TR7IBuffer"><code class="docutils literal notranslate"><span class="pre">bufferCast()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKN7IBuffer9SharedPtrE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKN7IBuffer14SharedConstPtrE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEP1TRKNSt8optionalIN7IBuffer9SharedPtrEEE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime16bufferCastOrNullEPK1TRKNSt8optionalIN7IBuffer14SharedConstPtrEEE"><code class="docutils literal notranslate"><span class="pre">bufferCastOrNull()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtimelsERNSt7ostreamERK7IBuffer"><code class="docutils literal notranslate"><span class="pre">operator&lt;&lt;()</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataTypeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::BufferDataType</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType14BufferDataTypeEN8nvinfer18DataTypeEbb"><code class="docutils literal notranslate"><span class="pre">BufferDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataTypecvN8nvinfer18DataTypeEEv"><code class="docutils literal notranslate"><span class="pre">operator</span> <span class="pre">nvinfer1::DataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType9isPointerEv"><code class="docutils literal notranslate"><span class="pre">isPointer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType10isUnsignedEv"><code class="docutils literal notranslate"><span class="pre">isUnsigned()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType7getSizeEv"><code class="docutils literal notranslate"><span class="pre">getSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime14BufferDataType13getSizeInBitsEv"><code class="docutils literal notranslate"><span class="pre">getSizeInBits()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType15kTrtPointerTypeE"><code class="docutils literal notranslate"><span class="pre">kTrtPointerType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType9mDataTypeE"><code class="docutils literal notranslate"><span class="pre">mDataType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType9mUnsignedE"><code class="docutils literal notranslate"><span class="pre">mUnsigned</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14BufferDataType8mPointerE"><code class="docutils literal notranslate"><span class="pre">mPointer</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime11BufferRangeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::BufferRange</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11BufferRange4BaseE"><code class="docutils literal notranslate"><span class="pre">Base</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11BufferRange11BufferRangeEP1T9size_type"><code class="docutils literal notranslate"><span class="pre">BufferRange()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tIXntNSt10is_const_vI1UEEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeER7IBuffer"><code class="docutils literal notranslate"><span class="pre">BufferRange()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI1UEEbEEEN12tensorrt_llm7runtime11BufferRange11BufferRangeERK7IBuffer"><code class="docutils literal notranslate"><span class="pre">BufferRange()</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_N8nvinfer18DataTypeE_b_bEN12tensorrt_llm7runtime14DataTypeTraitsE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_N8nvinfer18DataTypeE_bEN12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">kDataType,</span> <span class="pre">kUnsigned,</span> <span class="pre">true</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsI9kDataType9kUnsignedXL1EEE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kBOOL,</span> <span class="pre">kUnsigned</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kBOOLE9kUnsignedE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kFLOAT</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kFLOATEE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kHALF</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kHALFEE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kINT32</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kINT32,</span> <span class="pre">true</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT32EXL1EEE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kINT64</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kINT64,</span> <span class="pre">true</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kINT64EXL1EEE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kINT8</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType5kINT8EE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_bEN12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::DataTypeTraits&lt;</span> <span class="pre">nvinfer1::DataType::kUINT8,</span> <span class="pre">kUnsigned</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4typeE"><code class="docutils literal notranslate"><span class="pre">type</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4nameE"><code class="docutils literal notranslate"><span class="pre">name</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime14DataTypeTraitsIN8nvinfer18DataType6kUINT8E9kUnsignedE4sizeE"><code class="docutils literal notranslate"><span class="pre">size</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBufferE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::IBuffer</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9UniquePtrE"><code class="docutils literal notranslate"><span class="pre">UniquePtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer9SharedPtrE"><code class="docutils literal notranslate"><span class="pre">SharedPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14UniqueConstPtrE"><code class="docutils literal notranslate"><span class="pre">UniqueConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer14SharedConstPtrE"><code class="docutils literal notranslate"><span class="pre">SharedConstPtr</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer8DataTypeE"><code class="docutils literal notranslate"><span class="pre">DataType</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4dataEv"><code class="docutils literal notranslate"><span class="pre">data()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataEv"><code class="docutils literal notranslate"><span class="pre">data()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">data()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer4dataENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">data()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer7getSizeEv"><code class="docutils literal notranslate"><span class="pre">getSize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer14getSizeInBytesEv"><code class="docutils literal notranslate"><span class="pre">getSizeInBytes()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer11getCapacityEv"><code class="docutils literal notranslate"><span class="pre">getCapacity()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer11getDataTypeEv"><code class="docutils literal notranslate"><span class="pre">getDataType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer15getDataTypeNameEv"><code class="docutils literal notranslate"><span class="pre">getDataTypeName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer13getMemoryTypeEv"><code class="docutils literal notranslate"><span class="pre">getMemoryType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer17getMemoryTypeNameEv"><code class="docutils literal notranslate"><span class="pre">getMemoryTypeName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer6resizeENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">resize()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7releaseEv"><code class="docutils literal notranslate"><span class="pre">release()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBufferD0Ev"><code class="docutils literal notranslate"><span class="pre">~IBuffer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferERK7IBuffer"><code class="docutils literal notranslate"><span class="pre">IBuffer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBufferaSERK7IBuffer"><code class="docutils literal notranslate"><span class="pre">operator=()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer15getDataTypeNameE8DataType"><code class="docutils literal notranslate"><span class="pre">getDataTypeName()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer5sliceE9SharedPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer5sliceE14UniqueConstPtrRR9TConstPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">slice()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtr"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4viewE9SharedPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_NSt11enable_if_tINSt10is_const_vI18PointerElementTypeI9TConstPtrEEEiEEEN12tensorrt_llm7runtime7IBuffer4viewE14UniqueConstPtrRR9TConstPtrNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">view()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer4wrapEPv8DataTypeNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrP1TNSt6size_tE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime7IBuffer4wrapE9UniquePtrRNSt6vectorI1TEE"><code class="docutils literal notranslate"><span class="pre">wrap()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer10memoryTypeEPKv"><code class="docutils literal notranslate"><span class="pre">memoryType()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime7IBuffer7IBufferEv"><code class="docutils literal notranslate"><span class="pre">IBuffer()</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4NK12tensorrt_llm7runtime7IBuffer7toBytesENSt6size_tE"><code class="docutils literal notranslate"><span class="pre">toBytes()</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I_10MemoryTypeEN12tensorrt_llm7runtime16MemoryTypeStringE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString&lt;</span> <span class="pre">MemoryType::kCPU</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kCPUEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString&lt;</span> <span class="pre">MemoryType::kGPU</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kGPUEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString&lt;</span> <span class="pre">MemoryType::kPINNED</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType7kPINNEDEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString&lt;</span> <span class="pre">MemoryType::kPINNEDPOOL</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType11kPINNEDPOOLEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::MemoryTypeString&lt;</span> <span class="pre">MemoryType::kUVM</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime16MemoryTypeStringIN10MemoryType4kUVMEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0_bEN12tensorrt_llm7runtime11TRTDataTypeE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType</span></code></a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIbEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">bool</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIbE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIfEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">float</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIfE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeI4halfEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">half</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeI4halfE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">kernels::FinishedState</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels13FinishedStateEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">kernels::KVCacheIndex</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7kernels12KVCacheIndexEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">runtime::RequestType</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIN7runtime11RequestTypeEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::int32_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int32_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::int64_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7int64_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::int8_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt6int8_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::uint32_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint32_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::uint64_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt8uint64_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">std::uint8_t</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeINSt7uint8_tEE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4I0EN12tensorrt_llm7runtime11TRTDataTypeIP1TEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">T</span> <span class="pre">*</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIP1TE15kUnderlyingTypeE"><code class="docutils literal notranslate"><span class="pre">kUnderlyingType</span></code></a></li>
</ul>
</li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4IEN12tensorrt_llm7runtime11TRTDataTypeIPvEE"><code class="docutils literal notranslate"><span class="pre">tensorrt_llm::runtime::TRTDataType&lt;</span> <span class="pre">void</span> <span class="pre">*</span> <span class="pre">&gt;</span></code></a><ul class="nav section-nav flex-column">
<li class="toc-h4 nav-item toc-entry"><a class="reference internal nav-link" href="#_CPPv4N12tensorrt_llm7runtime11TRTDataTypeIPvE5valueE"><code class="docutils literal notranslate"><span class="pre">value</span></code></a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav></div>
</div></div>
</div>
<footer class="bd-footer-content">
</footer>
</main>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script defer src="../_static/scripts/bootstrap.js?digest=8878045cc6db502f8baf"></script>
<script defer src="../_static/scripts/pydata-sphinx-theme.js?digest=8878045cc6db502f8baf"></script>
<footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
<div class="footer-items__start">
<div class="footer-item">
<a class="footer-brand logo" href="https://www.nvidia.com">
<img src="../_static/nvidia-logo-horiz-rgb-1c-blk-for-screen.svg" class="logo__image only-light" alt="NVIDIA"/>
<img src="../_static/nvidia-logo-horiz-rgb-1c-wht-for-screen.svg" class="logo__image only-dark" alt="NVIDIA"/>
</a></div>
<div class="footer-item">
<div class="footer-links">
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-policy/">Privacy Policy</a>
|
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/privacy-center/">Manage My Privacy</a>
|
<a class="external" href="https://www.nvidia.com/en-us/preferences/start/">Do Not Sell or Share My Data</a>
|
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/terms-of-service/">Terms of Service</a>
|
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/accessibility/">Accessibility</a>
|
<a class="external" href="https://www.nvidia.com/en-us/about-nvidia/company-policies/">Corporate Policies</a>
|
<a class="external" href="https://www.nvidia.com/en-us/product-security/">Product Security</a>
|
<a class="external" href="https://www.nvidia.com/en-us/contact/">Contact</a>
</div>
</div>
<div class="footer-item">
<p class="copyright">
Copyright © 2025, NVidia.
<br/>
</p>
</div>
<div class="footer-item">
<div class="extra_footer">
<p>Last updated on June 19, 2025.</p>
<p>This page is generated by TensorRT-LLM commit <a href="https://github.com/NVIDIA/TensorRT-LLM/tree/47a765d">47a765d</a>.</p>
</div></div>
</div>
</div>
</footer>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink